2024-11-28 02:53:33,279 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-28 02:53:33,297 main DEBUG Took 0.015797 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-28 02:53:33,298 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-28 02:53:33,298 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-28 02:53:33,300 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-28 02:53:33,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:53:33,312 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-28 02:53:33,327 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,328 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:53:33,332 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,333 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:53:33,334 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,334 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:53:33,335 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:53:33,337 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,337 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:53:33,338 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,339 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:53:33,340 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,340 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:53:33,341 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,341 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:53:33,342 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,342 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:53:33,343 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,344 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:53:33,344 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,344 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:53:33,345 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,345 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:53:33,346 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,347 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-28 02:53:33,349 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:53:33,351 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-28 02:53:33,354 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-28 02:53:33,355 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-28 02:53:33,357 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-28 02:53:33,357 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-28 02:53:33,370 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-28 02:53:33,374 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-28 02:53:33,376 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-28 02:53:33,377 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-28 02:53:33,377 main DEBUG createAppenders(={Console}) 2024-11-28 02:53:33,379 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-28 02:53:33,379 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-28 02:53:33,380 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-28 02:53:33,381 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-28 02:53:33,381 main DEBUG OutputStream closed 2024-11-28 02:53:33,381 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-28 02:53:33,382 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-28 02:53:33,382 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-28 02:53:33,492 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-28 02:53:33,495 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-28 02:53:33,497 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-28 02:53:33,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-28 02:53:33,499 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-28 02:53:33,500 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-28 02:53:33,500 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-28 02:53:33,501 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-28 02:53:33,501 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-28 02:53:33,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-28 02:53:33,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-28 02:53:33,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-28 02:53:33,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-28 02:53:33,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-28 02:53:33,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-28 02:53:33,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-28 02:53:33,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-28 02:53:33,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-28 02:53:33,509 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28 02:53:33,510 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-28 02:53:33,510 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-28 02:53:33,511 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-28T02:53:33,820 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e 2024-11-28 02:53:33,824 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-28 02:53:33,825 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28T02:53:33,838 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-28T02:53:33,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-28T02:53:33,898 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/cluster_ae8f2237-5101-485d-8e1c-1b2973f77a47, deleteOnExit=true 2024-11-28T02:53:33,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-28T02:53:33,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/test.cache.data in system properties and HBase conf 2024-11-28T02:53:33,902 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/hadoop.tmp.dir in system properties and HBase conf 2024-11-28T02:53:33,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/hadoop.log.dir in system properties and HBase conf 2024-11-28T02:53:33,905 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-28T02:53:33,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-28T02:53:33,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-28T02:53:34,041 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-28T02:53:34,178 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T02:53:34,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-28T02:53:34,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-28T02:53:34,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-28T02:53:34,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T02:53:34,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-28T02:53:34,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-28T02:53:34,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T02:53:34,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T02:53:34,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-28T02:53:34,194 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/nfs.dump.dir in system properties and HBase conf 2024-11-28T02:53:34,194 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/java.io.tmpdir in system properties and HBase conf 2024-11-28T02:53:34,195 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T02:53:34,196 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-28T02:53:34,197 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-28T02:53:35,454 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-28T02:53:35,575 INFO [Time-limited test {}] log.Log(170): Logging initialized @3150ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-28T02:53:35,669 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T02:53:35,754 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T02:53:35,782 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T02:53:35,782 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T02:53:35,785 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T02:53:35,803 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T02:53:35,807 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/hadoop.log.dir/,AVAILABLE} 2024-11-28T02:53:35,808 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T02:53:36,038 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/java.io.tmpdir/jetty-localhost-43755-hadoop-hdfs-3_4_1-tests_jar-_-any-3725437006259365948/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T02:53:36,046 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:43755} 2024-11-28T02:53:36,047 INFO [Time-limited test {}] server.Server(415): Started @3623ms 2024-11-28T02:53:36,643 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T02:53:36,653 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T02:53:36,659 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T02:53:36,659 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T02:53:36,660 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-28T02:53:36,661 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/hadoop.log.dir/,AVAILABLE} 2024-11-28T02:53:36,662 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T02:53:36,790 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/java.io.tmpdir/jetty-localhost-38483-hadoop-hdfs-3_4_1-tests_jar-_-any-16037802612388364619/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T02:53:36,791 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:38483} 2024-11-28T02:53:36,791 INFO [Time-limited test {}] server.Server(415): Started @4367ms 2024-11-28T02:53:36,861 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-28T02:53:37,818 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/cluster_ae8f2237-5101-485d-8e1c-1b2973f77a47/data/data2/current/BP-872746337-172.17.0.2-1732762414943/current, will proceed with Du for space computation calculation, 2024-11-28T02:53:37,818 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/cluster_ae8f2237-5101-485d-8e1c-1b2973f77a47/data/data1/current/BP-872746337-172.17.0.2-1732762414943/current, will proceed with Du for space computation calculation, 2024-11-28T02:53:37,854 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-28T02:53:37,909 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6461fc71a53e6c27 with lease ID 0x76abcf41eb88a833: Processing first storage report for DS-9457f18b-b382-4192-9f46-ad0bcdc38b6a from datanode DatanodeRegistration(127.0.0.1:44799, datanodeUuid=a695dfb8-90fe-4dfb-9b42-b7f21223e480, infoPort=43107, infoSecurePort=0, ipcPort=36207, storageInfo=lv=-57;cid=testClusterID;nsid=280114296;c=1732762414943) 2024-11-28T02:53:37,911 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6461fc71a53e6c27 with lease ID 0x76abcf41eb88a833: from storage DS-9457f18b-b382-4192-9f46-ad0bcdc38b6a node DatanodeRegistration(127.0.0.1:44799, datanodeUuid=a695dfb8-90fe-4dfb-9b42-b7f21223e480, infoPort=43107, infoSecurePort=0, ipcPort=36207, storageInfo=lv=-57;cid=testClusterID;nsid=280114296;c=1732762414943), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-28T02:53:37,911 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6461fc71a53e6c27 with lease ID 0x76abcf41eb88a833: Processing first storage report for DS-0aa36455-70be-49c2-b02f-5a639ef1ddec from datanode DatanodeRegistration(127.0.0.1:44799, datanodeUuid=a695dfb8-90fe-4dfb-9b42-b7f21223e480, infoPort=43107, infoSecurePort=0, ipcPort=36207, storageInfo=lv=-57;cid=testClusterID;nsid=280114296;c=1732762414943) 2024-11-28T02:53:37,912 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6461fc71a53e6c27 with lease ID 0x76abcf41eb88a833: from storage DS-0aa36455-70be-49c2-b02f-5a639ef1ddec node DatanodeRegistration(127.0.0.1:44799, datanodeUuid=a695dfb8-90fe-4dfb-9b42-b7f21223e480, infoPort=43107, infoSecurePort=0, ipcPort=36207, storageInfo=lv=-57;cid=testClusterID;nsid=280114296;c=1732762414943), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-28T02:53:37,999 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e 2024-11-28T02:53:38,127 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/cluster_ae8f2237-5101-485d-8e1c-1b2973f77a47/zookeeper_0, clientPort=49459, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/cluster_ae8f2237-5101-485d-8e1c-1b2973f77a47/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/cluster_ae8f2237-5101-485d-8e1c-1b2973f77a47/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T02:53:38,140 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49459 2024-11-28T02:53:38,154 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:53:38,158 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:53:38,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741825_1001 (size=7) 2024-11-28T02:53:38,824 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381 with version=8 2024-11-28T02:53:38,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/hbase-staging 2024-11-28T02:53:38,924 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-28T02:53:39,196 INFO [Time-limited test {}] client.ConnectionUtils(128): master/978bd30859c3:0 server-side Connection retries=45 2024-11-28T02:53:39,207 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T02:53:39,207 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T02:53:39,212 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T02:53:39,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T02:53:39,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T02:53:39,385 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T02:53:39,462 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-28T02:53:39,473 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-28T02:53:39,477 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T02:53:39,506 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 103658 (auto-detected) 2024-11-28T02:53:39,507 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-28T02:53:39,527 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32823 2024-11-28T02:53:39,551 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:32823 connecting to ZooKeeper ensemble=127.0.0.1:49459 2024-11-28T02:53:39,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:328230x0, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T02:53:39,688 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:32823-0x10180a5ce790000 connected 2024-11-28T02:53:39,785 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:53:39,789 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:53:39,802 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T02:53:39,806 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381, hbase.cluster.distributed=false 2024-11-28T02:53:39,832 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T02:53:39,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32823 2024-11-28T02:53:39,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32823 2024-11-28T02:53:39,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32823 2024-11-28T02:53:39,846 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32823 2024-11-28T02:53:39,846 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32823 2024-11-28T02:53:39,996 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/978bd30859c3:0 server-side Connection retries=45 2024-11-28T02:53:39,999 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T02:53:39,999 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T02:53:39,999 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T02:53:39,999 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T02:53:40,000 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T02:53:40,003 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T02:53:40,007 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T02:53:40,009 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41631 2024-11-28T02:53:40,012 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41631 connecting to ZooKeeper ensemble=127.0.0.1:49459 2024-11-28T02:53:40,014 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:53:40,018 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:53:40,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:416310x0, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T02:53:40,053 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41631-0x10180a5ce790001 connected 2024-11-28T02:53:40,055 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T02:53:40,061 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T02:53:40,074 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T02:53:40,077 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T02:53:40,087 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T02:53:40,091 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41631 2024-11-28T02:53:40,097 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41631 2024-11-28T02:53:40,098 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41631 2024-11-28T02:53:40,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41631 2024-11-28T02:53:40,102 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41631 2024-11-28T02:53:40,124 DEBUG [M:0;978bd30859c3:32823 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;978bd30859c3:32823 2024-11-28T02:53:40,126 INFO [master/978bd30859c3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/978bd30859c3,32823,1732762419008 2024-11-28T02:53:40,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T02:53:40,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T02:53:40,161 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/978bd30859c3,32823,1732762419008 2024-11-28T02:53:40,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:53:40,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T02:53:40,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:53:40,203 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T02:53:40,204 INFO [master/978bd30859c3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/978bd30859c3,32823,1732762419008 from backup master directory 2024-11-28T02:53:40,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/978bd30859c3,32823,1732762419008 2024-11-28T02:53:40,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T02:53:40,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T02:53:40,217 WARN [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T02:53:40,217 INFO [master/978bd30859c3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=978bd30859c3,32823,1732762419008 2024-11-28T02:53:40,220 INFO [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-28T02:53:40,221 INFO [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-28T02:53:40,302 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/hbase.id] with ID: 7e36574f-6bcb-448f-b5da-ed8f9252fc52 2024-11-28T02:53:40,302 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/.tmp/hbase.id 2024-11-28T02:53:40,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741826_1002 (size=42) 2024-11-28T02:53:40,322 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/.tmp/hbase.id]:[hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/hbase.id] 2024-11-28T02:53:40,388 INFO [master/978bd30859c3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:53:40,396 INFO [master/978bd30859c3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T02:53:40,425 INFO [master/978bd30859c3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 27ms. 2024-11-28T02:53:40,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:53:40,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:53:40,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741827_1003 (size=196) 2024-11-28T02:53:40,500 INFO [master/978bd30859c3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:53:40,504 INFO [master/978bd30859c3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T02:53:40,525 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:40,531 INFO [master/978bd30859c3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T02:53:40,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741828_1004 (size=1189) 2024-11-28T02:53:40,991 INFO [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store 2024-11-28T02:53:41,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741829_1005 (size=34) 2024-11-28T02:53:41,414 INFO [master/978bd30859c3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-28T02:53:41,417 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:53:41,418 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T02:53:41,419 INFO [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:53:41,419 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:53:41,420 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T02:53:41,420 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:53:41,420 INFO [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:53:41,421 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732762421418Disabling compacts and flushes for region at 1732762421418Disabling writes for close at 1732762421420 (+2 ms)Writing region close event to WAL at 1732762421420Closed at 1732762421420 2024-11-28T02:53:41,423 WARN [master/978bd30859c3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/.initializing 2024-11-28T02:53:41,424 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/WALs/978bd30859c3,32823,1732762419008 2024-11-28T02:53:41,431 INFO [master/978bd30859c3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T02:53:41,445 INFO [master/978bd30859c3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=978bd30859c3%2C32823%2C1732762419008, suffix=, logDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/WALs/978bd30859c3,32823,1732762419008, archiveDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/oldWALs, maxLogs=10 2024-11-28T02:53:41,472 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/WALs/978bd30859c3,32823,1732762419008/978bd30859c3%2C32823%2C1732762419008.1732762421449, exclude list is [], retry=0 2024-11-28T02:53:41,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44799,DS-9457f18b-b382-4192-9f46-ad0bcdc38b6a,DISK] 2024-11-28T02:53:41,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-28T02:53:41,529 INFO [master/978bd30859c3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/WALs/978bd30859c3,32823,1732762419008/978bd30859c3%2C32823%2C1732762419008.1732762421449 2024-11-28T02:53:41,530 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43107:43107)] 2024-11-28T02:53:41,530 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:53:41,530 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:53:41,533 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:53:41,534 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:53:41,572 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:53:41,594 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T02:53:41,597 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:41,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:53:41,600 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:53:41,603 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T02:53:41,603 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:41,604 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:53:41,605 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:53:41,607 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T02:53:41,608 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:41,609 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:53:41,609 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:53:41,612 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T02:53:41,612 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:41,613 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:53:41,613 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:53:41,617 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:53:41,618 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:53:41,623 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:53:41,624 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:53:41,627 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T02:53:41,631 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:53:41,636 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:53:41,638 INFO [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71959117, jitterRate=0.07227440178394318}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T02:53:41,646 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732762421545Initializing all the Stores at 1732762421547 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732762421547Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762421548 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762421548Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762421548Cleaning up temporary data from old regions at 1732762421624 (+76 ms)Region opened successfully at 1732762421646 (+22 ms) 2024-11-28T02:53:41,648 INFO [master/978bd30859c3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T02:53:41,685 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58624a4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=978bd30859c3/172.17.0.2:0 2024-11-28T02:53:41,712 INFO [master/978bd30859c3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T02:53:41,721 INFO [master/978bd30859c3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T02:53:41,722 INFO [master/978bd30859c3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T02:53:41,724 INFO [master/978bd30859c3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T02:53:41,725 INFO [master/978bd30859c3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-28T02:53:41,730 INFO [master/978bd30859c3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-28T02:53:41,730 INFO [master/978bd30859c3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T02:53:41,753 INFO [master/978bd30859c3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T02:53:41,760 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T02:53:41,767 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T02:53:41,769 INFO [master/978bd30859c3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T02:53:41,771 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T02:53:41,782 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T02:53:41,785 INFO [master/978bd30859c3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T02:53:41,789 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T02:53:41,801 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T02:53:41,803 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T02:53:41,841 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T02:53:41,866 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T02:53:41,876 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T02:53:41,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T02:53:41,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T02:53:41,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:53:41,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:53:41,888 INFO [master/978bd30859c3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=978bd30859c3,32823,1732762419008, sessionid=0x10180a5ce790000, setting cluster-up flag (Was=false) 2024-11-28T02:53:41,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:53:41,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:53:41,943 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T02:53:41,945 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=978bd30859c3,32823,1732762419008 2024-11-28T02:53:41,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:53:41,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:53:42,001 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T02:53:42,003 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=978bd30859c3,32823,1732762419008 2024-11-28T02:53:42,010 INFO [master/978bd30859c3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T02:53:42,078 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T02:53:42,088 INFO [master/978bd30859c3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T02:53:42,093 INFO [master/978bd30859c3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T02:53:42,098 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 978bd30859c3,32823,1732762419008 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T02:53:42,105 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/978bd30859c3:0, corePoolSize=5, maxPoolSize=5 2024-11-28T02:53:42,105 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/978bd30859c3:0, corePoolSize=5, maxPoolSize=5 2024-11-28T02:53:42,106 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/978bd30859c3:0, corePoolSize=5, maxPoolSize=5 2024-11-28T02:53:42,106 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/978bd30859c3:0, corePoolSize=5, maxPoolSize=5 2024-11-28T02:53:42,106 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/978bd30859c3:0, corePoolSize=10, maxPoolSize=10 2024-11-28T02:53:42,106 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/978bd30859c3:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:53:42,107 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/978bd30859c3:0, corePoolSize=2, maxPoolSize=2 2024-11-28T02:53:42,107 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/978bd30859c3:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:53:42,109 INFO [master/978bd30859c3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732762452109 2024-11-28T02:53:42,110 INFO [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(746): ClusterId : 7e36574f-6bcb-448f-b5da-ed8f9252fc52 2024-11-28T02:53:42,112 INFO [master/978bd30859c3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T02:53:42,113 INFO [master/978bd30859c3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T02:53:42,113 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T02:53:42,115 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T02:53:42,115 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T02:53:42,117 INFO [master/978bd30859c3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T02:53:42,118 INFO [master/978bd30859c3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T02:53:42,118 INFO [master/978bd30859c3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T02:53:42,119 INFO [master/978bd30859c3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T02:53:42,120 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,122 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:42,123 INFO [PEWorker-2 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T02:53:42,125 INFO [master/978bd30859c3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T02:53:42,126 INFO [master/978bd30859c3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T02:53:42,127 INFO [master/978bd30859c3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T02:53:42,129 INFO [master/978bd30859c3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T02:53:42,129 INFO [master/978bd30859c3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T02:53:42,131 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/978bd30859c3:0:becomeActiveMaster-HFileCleaner.large.0-1732762422130,5,FailOnTimeoutGroup] 2024-11-28T02:53:42,134 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/978bd30859c3:0:becomeActiveMaster-HFileCleaner.small.0-1732762422131,5,FailOnTimeoutGroup] 2024-11-28T02:53:42,134 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,134 INFO [master/978bd30859c3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T02:53:42,136 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,137 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741831_1007 (size=1321) 2024-11-28T02:53:42,143 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T02:53:42,144 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T02:53:42,161 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T02:53:42,161 DEBUG [RS:0;978bd30859c3:41631 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57958468, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=978bd30859c3/172.17.0.2:0 2024-11-28T02:53:42,175 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;978bd30859c3:41631 2024-11-28T02:53:42,178 INFO [RS:0;978bd30859c3:41631 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T02:53:42,179 INFO [RS:0;978bd30859c3:41631 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T02:53:42,179 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T02:53:42,182 INFO [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(2659): reportForDuty to master=978bd30859c3,32823,1732762419008 with port=41631, startcode=1732762419946 2024-11-28T02:53:42,194 DEBUG [RS:0;978bd30859c3:41631 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T02:53:42,230 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49837, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T02:53:42,236 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32823 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 978bd30859c3,41631,1732762419946 2024-11-28T02:53:42,238 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32823 {}] master.ServerManager(517): Registering regionserver=978bd30859c3,41631,1732762419946 2024-11-28T02:53:42,251 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381 2024-11-28T02:53:42,251 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35935 2024-11-28T02:53:42,251 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T02:53:42,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T02:53:42,298 DEBUG [RS:0;978bd30859c3:41631 {}] zookeeper.ZKUtil(111): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/978bd30859c3,41631,1732762419946 2024-11-28T02:53:42,299 WARN [RS:0;978bd30859c3:41631 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T02:53:42,299 INFO [RS:0;978bd30859c3:41631 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T02:53:42,299 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/WALs/978bd30859c3,41631,1732762419946 2024-11-28T02:53:42,303 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [978bd30859c3,41631,1732762419946] 2024-11-28T02:53:42,329 INFO [RS:0;978bd30859c3:41631 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T02:53:42,347 INFO [RS:0;978bd30859c3:41631 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T02:53:42,352 INFO [RS:0;978bd30859c3:41631 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T02:53:42,352 INFO [RS:0;978bd30859c3:41631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,353 INFO [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T02:53:42,358 INFO [RS:0;978bd30859c3:41631 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T02:53:42,360 INFO [RS:0;978bd30859c3:41631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,360 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/978bd30859c3:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:53:42,360 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/978bd30859c3:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:53:42,360 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/978bd30859c3:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:53:42,360 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/978bd30859c3:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:53:42,360 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/978bd30859c3:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:53:42,360 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/978bd30859c3:0, corePoolSize=2, maxPoolSize=2 2024-11-28T02:53:42,361 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:53:42,361 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/978bd30859c3:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:53:42,361 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/978bd30859c3:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:53:42,361 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/978bd30859c3:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:53:42,361 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/978bd30859c3:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:53:42,362 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/978bd30859c3:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:53:42,362 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/978bd30859c3:0, corePoolSize=3, maxPoolSize=3 2024-11-28T02:53:42,362 DEBUG [RS:0;978bd30859c3:41631 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0, corePoolSize=3, maxPoolSize=3 2024-11-28T02:53:42,363 INFO [RS:0;978bd30859c3:41631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,364 INFO [RS:0;978bd30859c3:41631 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,364 INFO [RS:0;978bd30859c3:41631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,364 INFO [RS:0;978bd30859c3:41631 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,364 INFO [RS:0;978bd30859c3:41631 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,364 INFO [RS:0;978bd30859c3:41631 {}] hbase.ChoreService(168): Chore ScheduledChore name=978bd30859c3,41631,1732762419946-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T02:53:42,383 INFO [RS:0;978bd30859c3:41631 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T02:53:42,385 INFO [RS:0;978bd30859c3:41631 {}] hbase.ChoreService(168): Chore ScheduledChore name=978bd30859c3,41631,1732762419946-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,385 INFO [RS:0;978bd30859c3:41631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,386 INFO [RS:0;978bd30859c3:41631 {}] regionserver.Replication(171): 978bd30859c3,41631,1732762419946 started 2024-11-28T02:53:42,407 INFO [RS:0;978bd30859c3:41631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:42,407 INFO [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1482): Serving as 978bd30859c3,41631,1732762419946, RpcServer on 978bd30859c3/172.17.0.2:41631, sessionid=0x10180a5ce790001 2024-11-28T02:53:42,408 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T02:53:42,408 DEBUG [RS:0;978bd30859c3:41631 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 978bd30859c3,41631,1732762419946 2024-11-28T02:53:42,408 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '978bd30859c3,41631,1732762419946' 2024-11-28T02:53:42,409 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T02:53:42,410 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T02:53:42,410 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T02:53:42,411 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T02:53:42,411 DEBUG [RS:0;978bd30859c3:41631 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 978bd30859c3,41631,1732762419946 2024-11-28T02:53:42,411 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '978bd30859c3,41631,1732762419946' 2024-11-28T02:53:42,411 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T02:53:42,411 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T02:53:42,412 DEBUG [RS:0;978bd30859c3:41631 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T02:53:42,412 INFO [RS:0;978bd30859c3:41631 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T02:53:42,412 INFO [RS:0;978bd30859c3:41631 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T02:53:42,517 INFO [RS:0;978bd30859c3:41631 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T02:53:42,520 INFO [RS:0;978bd30859c3:41631 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=978bd30859c3%2C41631%2C1732762419946, suffix=, logDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/WALs/978bd30859c3,41631,1732762419946, archiveDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/oldWALs, maxLogs=32 2024-11-28T02:53:42,535 DEBUG [RS:0;978bd30859c3:41631 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/WALs/978bd30859c3,41631,1732762419946/978bd30859c3%2C41631%2C1732762419946.1732762422523, exclude list is [], retry=0 2024-11-28T02:53:42,539 INFO [PEWorker-2 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T02:53:42,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44799,DS-9457f18b-b382-4192-9f46-ad0bcdc38b6a,DISK] 2024-11-28T02:53:42,539 INFO [PEWorker-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381 2024-11-28T02:53:42,543 INFO [RS:0;978bd30859c3:41631 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/WALs/978bd30859c3,41631,1732762419946/978bd30859c3%2C41631%2C1732762419946.1732762422523 2024-11-28T02:53:42,544 DEBUG [RS:0;978bd30859c3:41631 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43107:43107)] 2024-11-28T02:53:42,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741833_1009 (size=32) 2024-11-28T02:53:42,955 DEBUG [PEWorker-2 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:53:42,958 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T02:53:42,960 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T02:53:42,961 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:42,962 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:53:42,962 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T02:53:42,965 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T02:53:42,965 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:42,966 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:53:42,966 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T02:53:42,969 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T02:53:42,970 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:42,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:53:42,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T02:53:42,973 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T02:53:42,973 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:42,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:53:42,974 DEBUG [PEWorker-2 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T02:53:42,976 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740 2024-11-28T02:53:42,977 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740 2024-11-28T02:53:42,979 DEBUG [PEWorker-2 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T02:53:42,980 DEBUG [PEWorker-2 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T02:53:42,981 DEBUG [PEWorker-2 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T02:53:42,984 DEBUG [PEWorker-2 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T02:53:42,988 DEBUG [PEWorker-2 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:53:42,989 INFO [PEWorker-2 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65801090, jitterRate=-0.019487351179122925}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T02:53:42,992 DEBUG [PEWorker-2 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732762422955Initializing all the Stores at 1732762422957 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732762422957Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732762422957Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762422957Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732762422958 (+1 ms)Cleaning up temporary data from old regions at 1732762422980 (+22 ms)Region opened successfully at 1732762422991 (+11 ms) 2024-11-28T02:53:42,992 DEBUG [PEWorker-2 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T02:53:42,992 INFO [PEWorker-2 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T02:53:42,992 DEBUG [PEWorker-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T02:53:42,992 DEBUG [PEWorker-2 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T02:53:42,992 DEBUG [PEWorker-2 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T02:53:42,994 INFO [PEWorker-2 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T02:53:42,994 DEBUG [PEWorker-2 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732762422992Disabling compacts and flushes for region at 1732762422992Disabling writes for close at 1732762422992Writing region close event to WAL at 1732762422993 (+1 ms)Closed at 1732762422993 2024-11-28T02:53:42,998 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T02:53:42,998 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T02:53:43,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T02:53:43,016 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T02:53:43,019 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T02:53:43,171 DEBUG [978bd30859c3:32823 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-28T02:53:43,181 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:53:43,186 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 978bd30859c3,41631,1732762419946, state=OPENING 2024-11-28T02:53:43,232 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T02:53:43,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:53:43,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:53:43,277 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T02:53:43,277 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T02:53:43,279 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T02:53:43,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:53:43,451 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T02:53:43,455 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56089, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T02:53:43,465 INFO [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T02:53:43,465 INFO [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T02:53:43,466 INFO [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T02:53:43,469 INFO [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=978bd30859c3%2C41631%2C1732762419946.meta, suffix=.meta, logDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/WALs/978bd30859c3,41631,1732762419946, archiveDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/oldWALs, maxLogs=32 2024-11-28T02:53:43,490 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/WALs/978bd30859c3,41631,1732762419946/978bd30859c3%2C41631%2C1732762419946.meta.1732762423471.meta, exclude list is [], retry=0 2024-11-28T02:53:43,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44799,DS-9457f18b-b382-4192-9f46-ad0bcdc38b6a,DISK] 2024-11-28T02:53:43,496 INFO [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/WALs/978bd30859c3,41631,1732762419946/978bd30859c3%2C41631%2C1732762419946.meta.1732762423471.meta 2024-11-28T02:53:43,497 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43107:43107)] 2024-11-28T02:53:43,497 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:53:43,499 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T02:53:43,502 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T02:53:43,506 INFO [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T02:53:43,510 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T02:53:43,511 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:53:43,511 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T02:53:43,511 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T02:53:43,514 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T02:53:43,516 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T02:53:43,516 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:43,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:53:43,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T02:53:43,519 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T02:53:43,519 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:43,520 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:53:43,520 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T02:53:43,522 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T02:53:43,522 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:43,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:53:43,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T02:53:43,524 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T02:53:43,524 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:43,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:53:43,525 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T02:53:43,527 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740 2024-11-28T02:53:43,529 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740 2024-11-28T02:53:43,532 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T02:53:43,532 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T02:53:43,533 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T02:53:43,535 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T02:53:43,537 INFO [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63499562, jitterRate=-0.05378279089927673}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T02:53:43,537 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T02:53:43,539 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732762423512Writing region info on filesystem at 1732762423512Initializing all the Stores at 1732762423514 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732762423514Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732762423514Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762423514Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732762423514Cleaning up temporary data from old regions at 1732762423532 (+18 ms)Running coprocessor post-open hooks at 1732762423537 (+5 ms)Region opened successfully at 1732762423539 (+2 ms) 2024-11-28T02:53:43,545 INFO [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732762423444 2024-11-28T02:53:43,556 DEBUG [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T02:53:43,556 INFO [RS_OPEN_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T02:53:43,558 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:53:43,561 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 978bd30859c3,41631,1732762419946, state=OPEN 2024-11-28T02:53:43,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T02:53:43,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T02:53:43,656 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T02:53:43,656 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T02:53:43,656 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:43,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T02:53:43,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=978bd30859c3,41631,1732762419946 in 376 msec 2024-11-28T02:53:43,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T02:53:43,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 658 msec 2024-11-28T02:53:43,671 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T02:53:43,671 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T02:53:43,689 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:53:43,690 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:53:43,710 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:53:43,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56583, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:53:43,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6920 sec 2024-11-28T02:53:43,733 INFO [master/978bd30859c3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732762423733, completionTime=-1 2024-11-28T02:53:43,736 INFO [master/978bd30859c3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-28T02:53:43,736 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T02:53:43,760 INFO [master/978bd30859c3:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-28T02:53:43,760 INFO [master/978bd30859c3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732762483760 2024-11-28T02:53:43,760 INFO [master/978bd30859c3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732762543760 2024-11-28T02:53:43,760 INFO [master/978bd30859c3:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-28T02:53:43,762 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=978bd30859c3,32823,1732762419008-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:43,762 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=978bd30859c3,32823,1732762419008-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:43,763 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=978bd30859c3,32823,1732762419008-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:43,764 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-978bd30859c3:32823, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:43,764 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:43,765 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:43,770 DEBUG [master/978bd30859c3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T02:53:43,794 INFO [master/978bd30859c3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.577sec 2024-11-28T02:53:43,795 INFO [master/978bd30859c3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T02:53:43,796 INFO [master/978bd30859c3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T02:53:43,797 INFO [master/978bd30859c3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T02:53:43,798 INFO [master/978bd30859c3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T02:53:43,798 INFO [master/978bd30859c3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T02:53:43,799 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=978bd30859c3,32823,1732762419008-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T02:53:43,799 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=978bd30859c3,32823,1732762419008-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T02:53:43,807 DEBUG [master/978bd30859c3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T02:53:43,808 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T02:53:43,808 INFO [master/978bd30859c3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=978bd30859c3,32823,1732762419008-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T02:53:43,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:43,819 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-28T02:53:43,819 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-28T02:53:43,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:53:43,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:53:43,831 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:53:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:53:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:53:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@386b99cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:53:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:53:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:53:43,840 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:53:43,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62702105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:43,843 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:53:43,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:53:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:53:43,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:53:43,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=978bd30859c3,32823,1732762419008 2024-11-28T02:53:43,891 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=352, ProcessCount=11, AvailableMemoryMB=7457 2024-11-28T02:53:43,899 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T02:53:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 978bd30859c3,32823,1732762419008 2024-11-28T02:53:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79901794 2024-11-28T02:53:43,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T02:53:43,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T02:53:43,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T02:53:43,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:53:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T02:53:43,928 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T02:53:43,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-28T02:53:43,931 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:43,948 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T02:53:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T02:53:43,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741835_1011 (size=1061) 2024-11-28T02:53:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T02:53:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T02:53:44,374 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381 2024-11-28T02:53:44,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741836_1012 (size=53) 2024-11-28T02:53:44,384 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:53:44,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1fec00bb9c12dd4e6236acd1fe2329f8, disabling compactions & flushes 2024-11-28T02:53:44,385 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:44,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:44,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. after waiting 0 ms 2024-11-28T02:53:44,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:44,385 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:44,385 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1fec00bb9c12dd4e6236acd1fe2329f8: Waiting for close lock at 1732762424385Disabling compacts and flushes for region at 1732762424385Disabling writes for close at 1732762424385Writing region close event to WAL at 1732762424385Closed at 1732762424385 2024-11-28T02:53:44,388 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T02:53:44,392 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732762424388"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732762424388"}]},"ts":"1732762424388"} 2024-11-28T02:53:44,397 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T02:53:44,398 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T02:53:44,401 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762424399"}]},"ts":"1732762424399"} 2024-11-28T02:53:44,405 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T02:53:44,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fec00bb9c12dd4e6236acd1fe2329f8, ASSIGN}] 2024-11-28T02:53:44,408 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fec00bb9c12dd4e6236acd1fe2329f8, ASSIGN 2024-11-28T02:53:44,410 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fec00bb9c12dd4e6236acd1fe2329f8, ASSIGN; state=OFFLINE, location=978bd30859c3,41631,1732762419946; forceNewPlan=false, retain=false 2024-11-28T02:53:44,562 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1fec00bb9c12dd4e6236acd1fe2329f8, regionState=OPENING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:53:44,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fec00bb9c12dd4e6236acd1fe2329f8, ASSIGN because future has completed 2024-11-28T02:53:44,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:53:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T02:53:44,730 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:44,731 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:53:44,732 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:44,732 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:53:44,732 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:44,732 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:44,736 INFO [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:44,740 INFO [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:53:44,740 INFO [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fec00bb9c12dd4e6236acd1fe2329f8 columnFamilyName A 2024-11-28T02:53:44,740 DEBUG [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:44,742 INFO [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] regionserver.HStore(327): Store=1fec00bb9c12dd4e6236acd1fe2329f8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:53:44,742 INFO [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:44,744 INFO [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:53:44,745 INFO [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fec00bb9c12dd4e6236acd1fe2329f8 columnFamilyName B 2024-11-28T02:53:44,745 DEBUG [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:44,746 INFO [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] regionserver.HStore(327): Store=1fec00bb9c12dd4e6236acd1fe2329f8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:53:44,746 INFO [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:44,749 INFO [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:53:44,749 INFO [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fec00bb9c12dd4e6236acd1fe2329f8 columnFamilyName C 2024-11-28T02:53:44,750 DEBUG [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:53:44,751 INFO [StoreOpener-1fec00bb9c12dd4e6236acd1fe2329f8-1 {}] regionserver.HStore(327): Store=1fec00bb9c12dd4e6236acd1fe2329f8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:53:44,752 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:44,752 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:44,754 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:44,755 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:44,757 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:44,757 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:44,758 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T02:53:44,761 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:44,765 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:53:44,766 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1fec00bb9c12dd4e6236acd1fe2329f8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65507564, jitterRate=-0.023861229419708252}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T02:53:44,766 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:44,767 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1fec00bb9c12dd4e6236acd1fe2329f8: Running coprocessor pre-open hook at 1732762424732Writing region info on filesystem at 1732762424732Initializing all the Stores at 1732762424734 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762424734Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762424736 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762424736Cleaning up temporary data from old regions at 1732762424757 (+21 ms)Running coprocessor post-open hooks at 1732762424766 (+9 ms)Region opened successfully at 1732762424767 (+1 ms) 2024-11-28T02:53:44,769 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., pid=6, masterSystemTime=1732762424722 2024-11-28T02:53:44,774 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:44,774 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:44,775 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1fec00bb9c12dd4e6236acd1fe2329f8, regionState=OPEN, openSeqNum=2, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:53:44,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:53:44,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T02:53:44,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 in 214 msec 2024-11-28T02:53:44,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T02:53:44,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fec00bb9c12dd4e6236acd1fe2329f8, ASSIGN in 381 msec 2024-11-28T02:53:44,795 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T02:53:44,795 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762424795"}]},"ts":"1732762424795"} 2024-11-28T02:53:44,799 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T02:53:44,802 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T02:53:44,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 881 msec 2024-11-28T02:53:45,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T02:53:45,088 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:53:45,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:53:45,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:53:45,100 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:53:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:53:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:53:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea8828e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:53:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:53:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:53:45,104 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:53:45,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d2b9a42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,106 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:53:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:53:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:53:45,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:53:45,115 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T02:53:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 978bd30859c3,32823,1732762419008 2024-11-28T02:53:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a040aee 2024-11-28T02:53:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T02:53:45,123 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T02:53:45,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:53:45,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:53:45,130 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:53:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:53:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:53:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd81cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:53:45,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:53:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:53:45,134 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:53:45,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a6e4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:53:45,140 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:53:45,142 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:53:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:53:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:53:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89c0433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:53:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:53:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:53:45,151 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:53:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef39c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:53:45,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:53:45,160 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:53:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:53:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:53:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7f3cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:53:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:53:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:53:45,164 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:53:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540312d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:53:45,167 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:53:45,169 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:53:45,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:53:45,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:53:45,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ee18a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:53:45,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:53:45,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:53:45,176 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:53:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@256affd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,179 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:53:45,179 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:53:45,180 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:53:45,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:53:45,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:53:45,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@106f4532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:53:45,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:53:45,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:53:45,184 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:53:45,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ccd238a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:53:45,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:53:45,188 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:53:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:53:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:53:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@294411fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:53:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:53:45,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:53:45,191 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:53:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bf87dca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:53:45,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:53:45,196 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:53:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:53:45,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:53:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1290d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:53:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:53:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:53:45,198 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:53:45,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175a1006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,202 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:53:45,202 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:53:45,204 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:53:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:53:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:53:45,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34feac5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:53:45,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:53:45,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:53:45,207 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:53:45,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48230300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:53:45,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:53:45,217 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:53:45,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:53:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:53:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2268467d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:53:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:53:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:53:45,221 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:53:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0f145b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:53:45,231 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:53:45,231 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:53:45,231 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:53:45,232 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:53:45,232 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:53:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:53:45,233 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:53:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:53:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:53:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:53:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:53:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:53:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:53:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:53:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:53:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:53:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:53:45,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:53:45,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:53:45,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:53:45,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:53:45,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:53:45,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:53:45,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:53:45,239 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:53:45,241 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:53:45,241 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:53:45,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:53:45,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:53:45,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:53:45,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:53:45,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:53:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:53:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:53:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:53:45,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:53:45,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:53:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-28T02:53:45,252 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:53:45,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:53:45,255 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:53:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T02:53:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:53:45,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:53:45,259 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:53:45,261 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:53:45,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:53:45,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:53:45,265 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:53:45,267 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:53:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:53:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:45,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:53:45,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:45,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:45,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:45,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:45,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:45,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T02:53:45,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T02:53:45,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:45,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:45,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:45,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:45,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:45,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/6d55513d0598428fa90b5b9952250792 is 50, key is test_row_0/A:col10/1732762425335/Put/seqid=0 2024-11-28T02:53:45,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762485490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:45,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762485490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:45,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762485491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:45,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762425467, max timestamp=1732762425495Num uniques 859; 2024-11-28T02:53:45,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762425467, max timestamp=1732762425495Num uniques 859; 2024-11-28T02:53:45,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:53:45,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762425467, max timestamp=1732762425495Num uniques 859; 2024-11-28T02:53:45,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:53:45,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:53:45,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741837_1013 (size=12001) 2024-11-28T02:53:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T02:53:45,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/6d55513d0598428fa90b5b9952250792 2024-11-28T02:53:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:45,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:45,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T02:53:45,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:45,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:45,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:45,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:45,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/b3644782f8874e069d2f7f351a8ee9a1 is 50, key is test_row_0/B:col10/1732762425335/Put/seqid=0 2024-11-28T02:53:45,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762485709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:45,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:45,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762485713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:45,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762485717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762485717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:45,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762485718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:45,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741838_1014 (size=12001) 2024-11-28T02:53:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:45,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T02:53:45,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:45,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:45,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:45,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:45,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:45,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762485827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:45,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762485827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T02:53:45,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762485937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:45,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762485938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:45,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762485938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:45,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T02:53:45,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:45,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:45,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:45,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:45,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762486046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762486047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T02:53:46,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:46,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:46,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:46,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/b3644782f8874e069d2f7f351a8ee9a1 2024-11-28T02:53:46,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/52d6dd8eb9434dadbb35e854e93d91f7 is 50, key is test_row_0/C:col10/1732762425335/Put/seqid=0 2024-11-28T02:53:46,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741839_1015 (size=12001) 2024-11-28T02:53:46,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/52d6dd8eb9434dadbb35e854e93d91f7 2024-11-28T02:53:46,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/6d55513d0598428fa90b5b9952250792 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6d55513d0598428fa90b5b9952250792 2024-11-28T02:53:46,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6d55513d0598428fa90b5b9952250792, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T02:53:46,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762486247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/b3644782f8874e069d2f7f351a8ee9a1 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b3644782f8874e069d2f7f351a8ee9a1 2024-11-28T02:53:46,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b3644782f8874e069d2f7f351a8ee9a1, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T02:53:46,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/52d6dd8eb9434dadbb35e854e93d91f7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/52d6dd8eb9434dadbb35e854e93d91f7 2024-11-28T02:53:46,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T02:53:46,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:46,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:46,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:46,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/52d6dd8eb9434dadbb35e854e93d91f7, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T02:53:46,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 947ms, sequenceid=12, compaction requested=false 2024-11-28T02:53:46,312 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T02:53:46,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:46,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-28T02:53:46,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:46,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:46,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:46,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:46,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:46,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:46,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/d0623743d0f944059e75fc2412c7052c is 50, key is test_row_0/A:col10/1732762425679/Put/seqid=0 2024-11-28T02:53:46,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741840_1016 (size=14341) 2024-11-28T02:53:46,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/d0623743d0f944059e75fc2412c7052c 2024-11-28T02:53:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T02:53:46,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/34b319e909364a80952f5e925097d0ee is 50, key is test_row_0/B:col10/1732762425679/Put/seqid=0 2024-11-28T02:53:46,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T02:53:46,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:46,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:46,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:46,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762486402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741841_1017 (size=14341) 2024-11-28T02:53:46,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762486438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762486444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762486446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762486547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T02:53:46,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:46,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:46,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762486588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762486589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762486589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T02:53:46,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:46,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:46,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:46,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762486757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762486776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/34b319e909364a80952f5e925097d0ee 2024-11-28T02:53:46,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762486848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762486847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762486848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:46,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/adb4907ea8e34c718969fae05f1ca2ea is 50, key is test_row_0/C:col10/1732762425679/Put/seqid=0 2024-11-28T02:53:46,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741842_1018 (size=12001) 2024-11-28T02:53:46,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/adb4907ea8e34c718969fae05f1ca2ea 2024-11-28T02:53:46,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T02:53:46,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:46,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:46,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:46,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:46,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/d0623743d0f944059e75fc2412c7052c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d0623743d0f944059e75fc2412c7052c 2024-11-28T02:53:46,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d0623743d0f944059e75fc2412c7052c, entries=200, sequenceid=49, filesize=14.0 K 2024-11-28T02:53:46,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/34b319e909364a80952f5e925097d0ee as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/34b319e909364a80952f5e925097d0ee 2024-11-28T02:53:46,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/34b319e909364a80952f5e925097d0ee, entries=200, sequenceid=49, filesize=14.0 K 2024-11-28T02:53:46,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/adb4907ea8e34c718969fae05f1ca2ea as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/adb4907ea8e34c718969fae05f1ca2ea 2024-11-28T02:53:47,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/adb4907ea8e34c718969fae05f1ca2ea, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T02:53:47,012 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-28T02:53:47,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 692ms, sequenceid=49, compaction requested=false 2024-11-28T02:53:47,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:47,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T02:53:47,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:47,076 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:53:47,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:47,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:47,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:47,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:47,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:47,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:47,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/5996478ad4604910bc1a9c42cf1f0008 is 50, key is test_row_0/A:col10/1732762426427/Put/seqid=0 2024-11-28T02:53:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:47,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:47,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741843_1019 (size=12001) 2024-11-28T02:53:47,170 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/5996478ad4604910bc1a9c42cf1f0008 2024-11-28T02:53:47,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/551aa9501fd242d481c720a93eec00e7 is 50, key is test_row_0/B:col10/1732762426427/Put/seqid=0 2024-11-28T02:53:47,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741844_1020 (size=12001) 2024-11-28T02:53:47,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762427225, max timestamp=1732762427252Num uniques 859; 2024-11-28T02:53:47,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:53:47,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762427225, max timestamp=1732762427252Num uniques 859; 2024-11-28T02:53:47,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:53:47,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762427225, max timestamp=1732762427252Num uniques 859; 2024-11-28T02:53:47,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:53:47,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762487299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762487303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762487309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762487311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T02:53:47,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762487419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762487419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762487437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762487437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762487627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762487628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,649 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/551aa9501fd242d481c720a93eec00e7 2024-11-28T02:53:47,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762487646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762487658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/52bb87d05c0141efa87297cee3a6f72e is 50, key is test_row_0/C:col10/1732762426427/Put/seqid=0 2024-11-28T02:53:47,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741845_1021 (size=12001) 2024-11-28T02:53:47,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762487787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,801 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 1fec00bb9c12dd4e6236acd1fe2329f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2308 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:53:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762487936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762487948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762487967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:47,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762487967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:47,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:48,119 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/52bb87d05c0141efa87297cee3a6f72e 2024-11-28T02:53:48,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/5996478ad4604910bc1a9c42cf1f0008 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/5996478ad4604910bc1a9c42cf1f0008 2024-11-28T02:53:48,155 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/5996478ad4604910bc1a9c42cf1f0008, entries=150, sequenceid=60, filesize=11.7 K 2024-11-28T02:53:48,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/551aa9501fd242d481c720a93eec00e7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/551aa9501fd242d481c720a93eec00e7 2024-11-28T02:53:48,171 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/551aa9501fd242d481c720a93eec00e7, entries=150, sequenceid=60, filesize=11.7 K 2024-11-28T02:53:48,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/52bb87d05c0141efa87297cee3a6f72e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/52bb87d05c0141efa87297cee3a6f72e 2024-11-28T02:53:48,188 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/52bb87d05c0141efa87297cee3a6f72e, entries=150, sequenceid=60, filesize=11.7 K 2024-11-28T02:53:48,190 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 1115ms, sequenceid=60, compaction requested=true 2024-11-28T02:53:48,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:48,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:48,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-28T02:53:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-28T02:53:48,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T02:53:48,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9370 sec 2024-11-28T02:53:48,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.9560 sec 2024-11-28T02:53:48,323 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-28T02:53:48,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-28T02:53:48,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:48,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:48,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:48,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:48,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:48,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:48,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/6ec6f71b95ce4a14810b9bf71993720b is 50, key is test_row_0/A:col10/1732762428462/Put/seqid=0 2024-11-28T02:53:48,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762488488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762488491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:48,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762488497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:48,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762488498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:48,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741846_1022 (size=12001) 2024-11-28T02:53:48,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762488606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:48,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762488617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:48,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:48,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762488617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:48,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762488816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:48,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:48,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762488827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:48,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:48,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762488827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:48,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:48,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/6ec6f71b95ce4a14810b9bf71993720b 2024-11-28T02:53:48,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/6be7b85ffc8840c988b1379f5940be45 is 50, key is test_row_0/B:col10/1732762428462/Put/seqid=0 2024-11-28T02:53:48,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741847_1023 (size=12001) 2024-11-28T02:53:48,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/6be7b85ffc8840c988b1379f5940be45 2024-11-28T02:53:49,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/ca8e6b9c9cc34f44834c262f58318e1f is 50, key is test_row_0/C:col10/1732762428462/Put/seqid=0 2024-11-28T02:53:49,027 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T02:53:49,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741848_1024 (size=12001) 2024-11-28T02:53:49,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/ca8e6b9c9cc34f44834c262f58318e1f 2024-11-28T02:53:49,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/6ec6f71b95ce4a14810b9bf71993720b as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6ec6f71b95ce4a14810b9bf71993720b 2024-11-28T02:53:49,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6ec6f71b95ce4a14810b9bf71993720b, entries=150, sequenceid=97, filesize=11.7 K 2024-11-28T02:53:49,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/6be7b85ffc8840c988b1379f5940be45 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6be7b85ffc8840c988b1379f5940be45 2024-11-28T02:53:49,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6be7b85ffc8840c988b1379f5940be45, entries=150, sequenceid=97, filesize=11.7 K 2024-11-28T02:53:49,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/ca8e6b9c9cc34f44834c262f58318e1f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/ca8e6b9c9cc34f44834c262f58318e1f 2024-11-28T02:53:49,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:49,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762489126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:49,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:49,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:49,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762489137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:49,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762489138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:49,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/ca8e6b9c9cc34f44834c262f58318e1f, entries=150, sequenceid=97, filesize=11.7 K 2024-11-28T02:53:49,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 701ms, sequenceid=97, compaction requested=true 2024-11-28T02:53:49,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:49,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:53:49,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:49,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:53:49,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:49,201 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:53:49,201 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:53:49,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:53:49,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:49,207 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:53:49,209 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:53:49,209 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/A is initiating minor compaction (all files) 2024-11-28T02:53:49,209 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/B is initiating minor compaction (all files) 2024-11-28T02:53:49,210 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/B in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:49,210 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/A in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:49,210 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b3644782f8874e069d2f7f351a8ee9a1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/34b319e909364a80952f5e925097d0ee, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/551aa9501fd242d481c720a93eec00e7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6be7b85ffc8840c988b1379f5940be45] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=49.2 K 2024-11-28T02:53:49,211 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6d55513d0598428fa90b5b9952250792, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d0623743d0f944059e75fc2412c7052c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/5996478ad4604910bc1a9c42cf1f0008, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6ec6f71b95ce4a14810b9bf71993720b] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=49.2 K 2024-11-28T02:53:49,214 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting b3644782f8874e069d2f7f351a8ee9a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732762425316 2024-11-28T02:53:49,215 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 34b319e909364a80952f5e925097d0ee, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732762425679 2024-11-28T02:53:49,216 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d55513d0598428fa90b5b9952250792, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732762425316 2024-11-28T02:53:49,217 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 551aa9501fd242d481c720a93eec00e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732762426427 2024-11-28T02:53:49,218 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6be7b85ffc8840c988b1379f5940be45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732762427297 2024-11-28T02:53:49,220 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0623743d0f944059e75fc2412c7052c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732762425679 2024-11-28T02:53:49,221 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5996478ad4604910bc1a9c42cf1f0008, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732762426427 2024-11-28T02:53:49,223 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ec6f71b95ce4a14810b9bf71993720b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732762427297 2024-11-28T02:53:49,272 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#B#compaction#12 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:49,273 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/1e73d010bdff443ba35096d065ca0ba9 is 50, key is test_row_0/B:col10/1732762428462/Put/seqid=0 2024-11-28T02:53:49,274 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#A#compaction#13 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:49,274 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/180b5fab41c449d78e638e182fbefd7c is 50, key is test_row_0/A:col10/1732762428462/Put/seqid=0 2024-11-28T02:53:49,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741849_1025 (size=12139) 2024-11-28T02:53:49,305 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/1e73d010bdff443ba35096d065ca0ba9 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/1e73d010bdff443ba35096d065ca0ba9 2024-11-28T02:53:49,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741850_1026 (size=12139) 2024-11-28T02:53:49,326 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/180b5fab41c449d78e638e182fbefd7c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/180b5fab41c449d78e638e182fbefd7c 2024-11-28T02:53:49,341 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/B of 1fec00bb9c12dd4e6236acd1fe2329f8 into 1e73d010bdff443ba35096d065ca0ba9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:49,341 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:49,344 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/A of 1fec00bb9c12dd4e6236acd1fe2329f8 into 180b5fab41c449d78e638e182fbefd7c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:49,344 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:49,346 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/A, priority=12, startTime=1732762429174; duration=0sec 2024-11-28T02:53:49,346 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/B, priority=12, startTime=1732762429201; duration=0sec 2024-11-28T02:53:49,346 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:49,347 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:49,347 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:A 2024-11-28T02:53:49,347 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:B 2024-11-28T02:53:49,347 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:53:49,351 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:53:49,351 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/C is initiating minor compaction (all files) 2024-11-28T02:53:49,351 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/C in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:49,351 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/52d6dd8eb9434dadbb35e854e93d91f7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/adb4907ea8e34c718969fae05f1ca2ea, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/52bb87d05c0141efa87297cee3a6f72e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/ca8e6b9c9cc34f44834c262f58318e1f] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=46.9 K 2024-11-28T02:53:49,353 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52d6dd8eb9434dadbb35e854e93d91f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732762425316 2024-11-28T02:53:49,354 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting adb4907ea8e34c718969fae05f1ca2ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732762425679 2024-11-28T02:53:49,356 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52bb87d05c0141efa87297cee3a6f72e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732762426427 2024-11-28T02:53:49,357 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca8e6b9c9cc34f44834c262f58318e1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732762427297 2024-11-28T02:53:49,397 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#C#compaction#14 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:49,398 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/36b3866f271c4bbb9f3e0f28714a6542 is 50, key is test_row_0/C:col10/1732762428462/Put/seqid=0 2024-11-28T02:53:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T02:53:49,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741851_1027 (size=12139) 2024-11-28T02:53:49,427 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:53:49,428 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:53:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-28T02:53:49,432 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:53:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T02:53:49,435 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:53:49,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:53:49,446 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/36b3866f271c4bbb9f3e0f28714a6542 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/36b3866f271c4bbb9f3e0f28714a6542 2024-11-28T02:53:49,458 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-28T02:53:49,458 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-28T02:53:49,461 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T02:53:49,461 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-28T02:53:49,461 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-28T02:53:49,461 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-28T02:53:49,463 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T02:53:49,463 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-28T02:53:49,471 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/C of 1fec00bb9c12dd4e6236acd1fe2329f8 into 36b3866f271c4bbb9f3e0f28714a6542(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:49,471 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:49,471 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/C, priority=12, startTime=1732762429201; duration=0sec 2024-11-28T02:53:49,472 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:49,472 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:C 2024-11-28T02:53:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:49,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T02:53:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:49,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/2e40fa16aa2f415cb4777897ff7c0f75 is 50, key is test_row_0/A:col10/1732762429508/Put/seqid=0 2024-11-28T02:53:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T02:53:49,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741852_1028 (size=14341) 2024-11-28T02:53:49,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/2e40fa16aa2f415cb4777897ff7c0f75 2024-11-28T02:53:49,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T02:53:49,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:49,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:49,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:49,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:49,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:49,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/25502e79940e46e3931e4155c8671dc4 is 50, key is test_row_0/B:col10/1732762429508/Put/seqid=0 2024-11-28T02:53:49,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741853_1029 (size=12001) 2024-11-28T02:53:49,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762429629, max timestamp=1732762429682Num uniques 859; 2024-11-28T02:53:49,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:53:49,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732762429629, max timestamp=1732762429682Num uniques 859; 2024-11-28T02:53:49,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-28T02:53:49,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762429629, max timestamp=1732762429689Num uniques 859; 2024-11-28T02:53:49,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:53:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T02:53:49,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T02:53:49,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:49,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:49,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:49,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:49,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:49,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762489785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:49,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762489785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:49,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762489788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:49,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762489793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:49,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762489807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:49,810 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 1fec00bb9c12dd4e6236acd1fe2329f8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4321 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:53:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:49,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T02:53:49,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:49,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:49,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:49,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:49,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:49,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762489907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:49,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762489907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:49,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762489908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:49,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762489917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T02:53:50,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T02:53:50,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:50,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:50,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:50,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:50,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:50,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/25502e79940e46e3931e4155c8671dc4 2024-11-28T02:53:50,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/7f222cef905a4b96bb93b5daccfb24a6 is 50, key is test_row_0/C:col10/1732762429508/Put/seqid=0 2024-11-28T02:53:50,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741854_1030 (size=12001) 2024-11-28T02:53:50,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/7f222cef905a4b96bb93b5daccfb24a6 2024-11-28T02:53:50,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/2e40fa16aa2f415cb4777897ff7c0f75 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2e40fa16aa2f415cb4777897ff7c0f75 2024-11-28T02:53:50,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762490127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2e40fa16aa2f415cb4777897ff7c0f75, entries=200, sequenceid=110, filesize=14.0 K 2024-11-28T02:53:50,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/25502e79940e46e3931e4155c8671dc4 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/25502e79940e46e3931e4155c8671dc4 2024-11-28T02:53:50,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762490128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762490128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762490128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/25502e79940e46e3931e4155c8671dc4, entries=150, sequenceid=110, filesize=11.7 K 2024-11-28T02:53:50,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/7f222cef905a4b96bb93b5daccfb24a6 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/7f222cef905a4b96bb93b5daccfb24a6 2024-11-28T02:53:50,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T02:53:50,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:50,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:50,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:50,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:50,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:50,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:50,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/7f222cef905a4b96bb93b5daccfb24a6, entries=150, sequenceid=110, filesize=11.7 K 2024-11-28T02:53:50,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 712ms, sequenceid=110, compaction requested=false 2024-11-28T02:53:50,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:50,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T02:53:50,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:50,375 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-11-28T02:53:50,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:50,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:50,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:50,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:50,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:50,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:50,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/166812658f9a49f69c5b2a736a06a4b4 is 50, key is test_row_0/A:col10/1732762429753/Put/seqid=0 2024-11-28T02:53:50,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741855_1031 (size=12151) 2024-11-28T02:53:50,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:50,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:50,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762490458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762490458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762490462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762490462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T02:53:50,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762490577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762490577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762490577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762490586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762490786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762490787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762490787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762490806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:50,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:50,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:50,841 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/166812658f9a49f69c5b2a736a06a4b4 2024-11-28T02:53:50,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/faa18605e45643f29983c731730bc035 is 50, key is test_row_0/B:col10/1732762429753/Put/seqid=0 2024-11-28T02:53:50,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741856_1032 (size=12151) 2024-11-28T02:53:51,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762491096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762491096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:51,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:51,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762491096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:51,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:51,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:51,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762491116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:51,280 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/faa18605e45643f29983c731730bc035 2024-11-28T02:53:51,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/4bb2f93c1a0d4ca99930dc736096152a is 50, key is test_row_0/C:col10/1732762429753/Put/seqid=0 2024-11-28T02:53:51,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741857_1033 (size=12151) 2024-11-28T02:53:51,309 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/4bb2f93c1a0d4ca99930dc736096152a 2024-11-28T02:53:51,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/166812658f9a49f69c5b2a736a06a4b4 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/166812658f9a49f69c5b2a736a06a4b4 2024-11-28T02:53:51,335 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/166812658f9a49f69c5b2a736a06a4b4, entries=150, sequenceid=147, filesize=11.9 K 2024-11-28T02:53:51,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/faa18605e45643f29983c731730bc035 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/faa18605e45643f29983c731730bc035 2024-11-28T02:53:51,357 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/faa18605e45643f29983c731730bc035, entries=150, sequenceid=147, filesize=11.9 K 2024-11-28T02:53:51,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/4bb2f93c1a0d4ca99930dc736096152a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4bb2f93c1a0d4ca99930dc736096152a 2024-11-28T02:53:51,378 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4bb2f93c1a0d4ca99930dc736096152a, entries=150, sequenceid=147, filesize=11.9 K 2024-11-28T02:53:51,380 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=46.96 KB/48090 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 1005ms, sequenceid=147, compaction requested=true 2024-11-28T02:53:51,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:51,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:51,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-28T02:53:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-28T02:53:51,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-28T02:53:51,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9490 sec 2024-11-28T02:53:51,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.9620 sec 2024-11-28T02:53:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T02:53:51,577 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:53:51,577 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:53:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-28T02:53:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T02:53:51,580 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:53:51,581 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:53:51,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:53:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:51,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:53:51,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:51,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:51,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/eae2c35a1e9649318d256cf682a33206 is 50, key is test_row_0/A:col10/1732762431607/Put/seqid=0 2024-11-28T02:53:51,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741858_1034 (size=12151) 2024-11-28T02:53:51,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/eae2c35a1e9649318d256cf682a33206 2024-11-28T02:53:51,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/38992963d147417db213e2d2ff2f0359 is 50, key is test_row_0/B:col10/1732762431607/Put/seqid=0 2024-11-28T02:53:51,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741859_1035 (size=12151) 2024-11-28T02:53:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T02:53:51,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732762431650, max timestamp=1732762431694Num uniques 859; 2024-11-28T02:53:51,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T02:53:51,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732762431650, max timestamp=1732762431694Num uniques 859; 2024-11-28T02:53:51,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T02:53:51,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732762431650, max timestamp=1732762431694Num uniques 859; 2024-11-28T02:53:51,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T02:53:51,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T02:53:51,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:51,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:51,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:51,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:51,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:51,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762491737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:51,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762491741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762491745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:51,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762491754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:51,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762491856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:51,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762491866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762491866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:51,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762491866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:51,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:51,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T02:53:51,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:51,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:51,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:51,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:51,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T02:53:52,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T02:53:52,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:52,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:52,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:52,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:52,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:52,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762492067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:52,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/38992963d147417db213e2d2ff2f0359 2024-11-28T02:53:52,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:52,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:52,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762492076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762492076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762492077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:52,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:52,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/19e3fc4ea94748f3a18efc436e9e75de is 50, key is test_row_0/C:col10/1732762431607/Put/seqid=0 2024-11-28T02:53:52,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741860_1036 (size=12151) 2024-11-28T02:53:52,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T02:53:52,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:52,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:52,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:52,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:52,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T02:53:52,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T02:53:52,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:52,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:52,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:52,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:52,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:52,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762492376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:52,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762492396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:52,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762492397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:52,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762492397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:52,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:52,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/19e3fc4ea94748f3a18efc436e9e75de 2024-11-28T02:53:52,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T02:53:52,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:52,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:52,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:52,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:52,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:52,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/eae2c35a1e9649318d256cf682a33206 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/eae2c35a1e9649318d256cf682a33206 2024-11-28T02:53:52,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/eae2c35a1e9649318d256cf682a33206, entries=150, sequenceid=158, filesize=11.9 K 2024-11-28T02:53:52,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/38992963d147417db213e2d2ff2f0359 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/38992963d147417db213e2d2ff2f0359 2024-11-28T02:53:52,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/38992963d147417db213e2d2ff2f0359, entries=150, sequenceid=158, filesize=11.9 K 2024-11-28T02:53:52,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/19e3fc4ea94748f3a18efc436e9e75de as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/19e3fc4ea94748f3a18efc436e9e75de 2024-11-28T02:53:52,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/19e3fc4ea94748f3a18efc436e9e75de, entries=150, sequenceid=158, filesize=11.9 K 2024-11-28T02:53:52,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 943ms, sequenceid=158, compaction requested=true 2024-11-28T02:53:52,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:52,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:53:52,553 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:53:52,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:52,553 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:53:52,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:53:52,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:52,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:53:52,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:52,556 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:53:52,556 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:53:52,556 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/A is initiating minor compaction (all files) 2024-11-28T02:53:52,556 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/B is initiating minor compaction (all files) 2024-11-28T02:53:52,556 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/B in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:52,556 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/A in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:52,556 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/1e73d010bdff443ba35096d065ca0ba9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/25502e79940e46e3931e4155c8671dc4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/faa18605e45643f29983c731730bc035, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/38992963d147417db213e2d2ff2f0359] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=47.3 K 2024-11-28T02:53:52,556 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/180b5fab41c449d78e638e182fbefd7c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2e40fa16aa2f415cb4777897ff7c0f75, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/166812658f9a49f69c5b2a736a06a4b4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/eae2c35a1e9649318d256cf682a33206] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=49.6 K 2024-11-28T02:53:52,557 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e73d010bdff443ba35096d065ca0ba9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732762427297 2024-11-28T02:53:52,557 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 180b5fab41c449d78e638e182fbefd7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732762427297 2024-11-28T02:53:52,560 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 25502e79940e46e3931e4155c8671dc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762428490 2024-11-28T02:53:52,561 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e40fa16aa2f415cb4777897ff7c0f75, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762428490 2024-11-28T02:53:52,562 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting faa18605e45643f29983c731730bc035, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732762429753 2024-11-28T02:53:52,562 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 166812658f9a49f69c5b2a736a06a4b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732762429753 2024-11-28T02:53:52,563 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 38992963d147417db213e2d2ff2f0359, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732762430457 2024-11-28T02:53:52,563 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting eae2c35a1e9649318d256cf682a33206, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732762430457 2024-11-28T02:53:52,585 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#A#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:52,585 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#B#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:52,586 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/4ed297548fe44febaf84807894766b73 is 50, key is test_row_0/A:col10/1732762431607/Put/seqid=0 2024-11-28T02:53:52,587 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/cd7c186d06d941f989f6f4a68519b327 is 50, key is test_row_0/B:col10/1732762431607/Put/seqid=0 2024-11-28T02:53:52,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741862_1038 (size=12425) 2024-11-28T02:53:52,607 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/cd7c186d06d941f989f6f4a68519b327 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/cd7c186d06d941f989f6f4a68519b327 2024-11-28T02:53:52,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741861_1037 (size=12425) 2024-11-28T02:53:52,622 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/B of 1fec00bb9c12dd4e6236acd1fe2329f8 into cd7c186d06d941f989f6f4a68519b327(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:52,622 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:52,622 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/B, priority=12, startTime=1732762432553; duration=0sec 2024-11-28T02:53:52,623 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:52,623 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:B 2024-11-28T02:53:52,623 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:53:52,626 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:53:52,627 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/C is initiating minor compaction (all files) 2024-11-28T02:53:52,627 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/C in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:52,627 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/36b3866f271c4bbb9f3e0f28714a6542, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/7f222cef905a4b96bb93b5daccfb24a6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4bb2f93c1a0d4ca99930dc736096152a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/19e3fc4ea94748f3a18efc436e9e75de] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=47.3 K 2024-11-28T02:53:52,628 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b3866f271c4bbb9f3e0f28714a6542, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732762427297 2024-11-28T02:53:52,629 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f222cef905a4b96bb93b5daccfb24a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762428490 2024-11-28T02:53:52,630 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb2f93c1a0d4ca99930dc736096152a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732762429753 2024-11-28T02:53:52,630 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 19e3fc4ea94748f3a18efc436e9e75de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732762430457 2024-11-28T02:53:52,655 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:52,656 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/b41dd0f8adb84cf4a5376d158413cc37 is 50, key is test_row_0/C:col10/1732762431607/Put/seqid=0 2024-11-28T02:53:52,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741863_1039 (size=12425) 2024-11-28T02:53:52,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T02:53:52,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:52,668 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-28T02:53:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:52,689 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/b41dd0f8adb84cf4a5376d158413cc37 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b41dd0f8adb84cf4a5376d158413cc37 2024-11-28T02:53:52,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/3c1b0fbc4d444755a315b4d56cdbafa4 is 50, key is test_row_0/A:col10/1732762431750/Put/seqid=0 2024-11-28T02:53:52,710 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/C of 1fec00bb9c12dd4e6236acd1fe2329f8 into b41dd0f8adb84cf4a5376d158413cc37(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:52,710 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:52,710 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/C, priority=12, startTime=1732762432554; duration=0sec 2024-11-28T02:53:52,711 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:52,712 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:C 2024-11-28T02:53:52,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T02:53:52,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741864_1040 (size=12151) 2024-11-28T02:53:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:52,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:52,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762492919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:52,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762492930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:52,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762492933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:52,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762492947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:52,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:52,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,027 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/4ed297548fe44febaf84807894766b73 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/4ed297548fe44febaf84807894766b73 2024-11-28T02:53:53,037 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/A of 1fec00bb9c12dd4e6236acd1fe2329f8 into 4ed297548fe44febaf84807894766b73(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:53,037 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:53,037 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/A, priority=12, startTime=1732762432553; duration=0sec 2024-11-28T02:53:53,038 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:53,038 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:A 2024-11-28T02:53:53,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762493036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762493056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762493056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762493056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,125 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/3c1b0fbc4d444755a315b4d56cdbafa4 2024-11-28T02:53:53,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/6bf6ba2bf67043e1bcc24a0e3c3420e7 is 50, key is test_row_0/B:col10/1732762431750/Put/seqid=0 2024-11-28T02:53:53,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741865_1041 (size=12151) 2024-11-28T02:53:53,164 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/6bf6ba2bf67043e1bcc24a0e3c3420e7 2024-11-28T02:53:53,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/7e3910b920e44f6590eeb5993f4aa791 is 50, key is test_row_0/C:col10/1732762431750/Put/seqid=0 2024-11-28T02:53:53,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762493246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741866_1042 (size=12151) 2024-11-28T02:53:53,261 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/7e3910b920e44f6590eeb5993f4aa791 2024-11-28T02:53:53,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762493267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762493267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762493267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/3c1b0fbc4d444755a315b4d56cdbafa4 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/3c1b0fbc4d444755a315b4d56cdbafa4 2024-11-28T02:53:53,283 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/3c1b0fbc4d444755a315b4d56cdbafa4, entries=150, sequenceid=193, filesize=11.9 K 2024-11-28T02:53:53,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/6bf6ba2bf67043e1bcc24a0e3c3420e7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6bf6ba2bf67043e1bcc24a0e3c3420e7 2024-11-28T02:53:53,306 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6bf6ba2bf67043e1bcc24a0e3c3420e7, entries=150, sequenceid=193, filesize=11.9 K 2024-11-28T02:53:53,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/7e3910b920e44f6590eeb5993f4aa791 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/7e3910b920e44f6590eeb5993f4aa791 2024-11-28T02:53:53,321 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/7e3910b920e44f6590eeb5993f4aa791, entries=150, sequenceid=193, filesize=11.9 K 2024-11-28T02:53:53,324 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=60.38 KB/61830 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 655ms, sequenceid=193, compaction requested=false 2024-11-28T02:53:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-28T02:53:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-28T02:53:53,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-28T02:53:53,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7470 sec 2024-11-28T02:53:53,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.7600 sec 2024-11-28T02:53:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:53,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T02:53:53,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:53,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:53,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:53,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:53,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:53,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:53,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/9802668b56524209924414ad01ed63c0 is 50, key is test_row_0/A:col10/1732762432907/Put/seqid=0 2024-11-28T02:53:53,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741867_1043 (size=14541) 2024-11-28T02:53:53,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762493595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762493598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762493600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762493602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762493706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762493716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762493716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762493716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T02:53:53,726 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:53:53,727 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:53:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-28T02:53:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T02:53:53,729 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:53:53,731 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:53:53,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:53:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T02:53:53,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762493856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,858 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 1fec00bb9c12dd4e6236acd1fe2329f8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8369 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:53:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T02:53:53,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:53,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:53,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:53,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:53,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:53,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762493916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762493926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762493936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762493937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:53,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/9802668b56524209924414ad01ed63c0 2024-11-28T02:53:53,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/26aea37ed6b44fcda446d7028fdda2ca is 50, key is test_row_0/B:col10/1732762432907/Put/seqid=0 2024-11-28T02:53:54,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741868_1044 (size=12151) 2024-11-28T02:53:54,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T02:53:54,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:54,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:54,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:54,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:54,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T02:53:54,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T02:53:54,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:54,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:54,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:54,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:54,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:54,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762494226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:54,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762494235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:54,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762494246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:54,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762494246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:54,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T02:53:54,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:54,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:54,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:54,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:54,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T02:53:54,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/26aea37ed6b44fcda446d7028fdda2ca 2024-11-28T02:53:54,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/4111ce771aed48f296efdad4dbd656e2 is 50, key is test_row_0/C:col10/1732762432907/Put/seqid=0 2024-11-28T02:53:54,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741869_1045 (size=12151) 2024-11-28T02:53:54,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/4111ce771aed48f296efdad4dbd656e2 2024-11-28T02:53:54,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/9802668b56524209924414ad01ed63c0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/9802668b56524209924414ad01ed63c0 2024-11-28T02:53:54,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/9802668b56524209924414ad01ed63c0, entries=200, sequenceid=208, filesize=14.2 K 2024-11-28T02:53:54,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/26aea37ed6b44fcda446d7028fdda2ca as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/26aea37ed6b44fcda446d7028fdda2ca 2024-11-28T02:53:54,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/26aea37ed6b44fcda446d7028fdda2ca, entries=150, sequenceid=208, filesize=11.9 K 2024-11-28T02:53:54,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/4111ce771aed48f296efdad4dbd656e2 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4111ce771aed48f296efdad4dbd656e2 2024-11-28T02:53:54,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T02:53:54,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:54,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:54,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:54,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:54,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:54,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4111ce771aed48f296efdad4dbd656e2, entries=150, sequenceid=208, filesize=11.9 K 2024-11-28T02:53:54,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 970ms, sequenceid=208, compaction requested=true 2024-11-28T02:53:54,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:54,533 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:53:54,536 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:53:54,536 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/A is initiating minor compaction (all files) 2024-11-28T02:53:54,536 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/A in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:54,537 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/4ed297548fe44febaf84807894766b73, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/3c1b0fbc4d444755a315b4d56cdbafa4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/9802668b56524209924414ad01ed63c0] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=38.2 K 2024-11-28T02:53:54,538 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ed297548fe44febaf84807894766b73, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732762430457 2024-11-28T02:53:54,539 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c1b0fbc4d444755a315b4d56cdbafa4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732762431724 2024-11-28T02:53:54,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:53:54,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:54,539 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:53:54,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:53:54,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:54,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:53:54,539 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9802668b56524209924414ad01ed63c0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732762432907 2024-11-28T02:53:54,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:54,543 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:53:54,544 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/B is initiating minor compaction (all files) 2024-11-28T02:53:54,544 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/B in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:54,544 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/cd7c186d06d941f989f6f4a68519b327, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6bf6ba2bf67043e1bcc24a0e3c3420e7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/26aea37ed6b44fcda446d7028fdda2ca] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=35.9 K 2024-11-28T02:53:54,546 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting cd7c186d06d941f989f6f4a68519b327, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732762430457 2024-11-28T02:53:54,547 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bf6ba2bf67043e1bcc24a0e3c3420e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732762431724 2024-11-28T02:53:54,550 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 26aea37ed6b44fcda446d7028fdda2ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732762432907 2024-11-28T02:53:54,574 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:54,575 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#A#compaction#33 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:54,575 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/023d301f9755483dab67b4a8ca8b358f is 50, key is test_row_0/A:col10/1732762432907/Put/seqid=0 2024-11-28T02:53:54,575 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/c0d47269dd0f4233b01d52019292d831 is 50, key is test_row_0/B:col10/1732762432907/Put/seqid=0 2024-11-28T02:53:54,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741871_1047 (size=12527) 2024-11-28T02:53:54,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741870_1046 (size=12527) 2024-11-28T02:53:54,613 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/023d301f9755483dab67b4a8ca8b358f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/023d301f9755483dab67b4a8ca8b358f 2024-11-28T02:53:54,622 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/A of 1fec00bb9c12dd4e6236acd1fe2329f8 into 023d301f9755483dab67b4a8ca8b358f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:54,622 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:54,622 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/A, priority=13, startTime=1732762434533; duration=0sec 2024-11-28T02:53:54,623 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:54,623 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:A 2024-11-28T02:53:54,623 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:53:54,627 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:53:54,628 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/C is initiating minor compaction (all files) 2024-11-28T02:53:54,628 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/C in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:54,628 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b41dd0f8adb84cf4a5376d158413cc37, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/7e3910b920e44f6590eeb5993f4aa791, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4111ce771aed48f296efdad4dbd656e2] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=35.9 K 2024-11-28T02:53:54,628 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b41dd0f8adb84cf4a5376d158413cc37, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732762430457 2024-11-28T02:53:54,629 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e3910b920e44f6590eeb5993f4aa791, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732762431724 2024-11-28T02:53:54,629 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4111ce771aed48f296efdad4dbd656e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732762432907 2024-11-28T02:53:54,645 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#C#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:54,645 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/43658e1ceb674cce9d5cb3c463f0b14e is 50, key is test_row_0/C:col10/1732762432907/Put/seqid=0 2024-11-28T02:53:54,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741872_1048 (size=12527) 2024-11-28T02:53:54,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T02:53:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:54,665 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T02:53:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:54,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:54,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:54,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:54,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:54,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:54,667 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/43658e1ceb674cce9d5cb3c463f0b14e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/43658e1ceb674cce9d5cb3c463f0b14e 2024-11-28T02:53:54,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/995ddb1afa1144c0a2d2c072f0d04d0b is 50, key is test_row_0/A:col10/1732762433599/Put/seqid=0 2024-11-28T02:53:54,683 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/C of 1fec00bb9c12dd4e6236acd1fe2329f8 into 43658e1ceb674cce9d5cb3c463f0b14e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:54,683 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:54,684 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/C, priority=13, startTime=1732762434539; duration=0sec 2024-11-28T02:53:54,684 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:54,684 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:C 2024-11-28T02:53:54,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741873_1049 (size=12151) 2024-11-28T02:53:54,690 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/995ddb1afa1144c0a2d2c072f0d04d0b 2024-11-28T02:53:54,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/ca5b2179b73f4925b4da78b6112a5485 is 50, key is test_row_0/B:col10/1732762433599/Put/seqid=0 2024-11-28T02:53:54,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741874_1050 (size=12151) 2024-11-28T02:53:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:54,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:54,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762494763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:54,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762494764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762494766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:54,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762494768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T02:53:54,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762494876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:54,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762494877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762494877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:54,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762494877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,017 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/c0d47269dd0f4233b01d52019292d831 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/c0d47269dd0f4233b01d52019292d831 2024-11-28T02:53:55,027 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/B of 1fec00bb9c12dd4e6236acd1fe2329f8 into c0d47269dd0f4233b01d52019292d831(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:55,027 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:55,027 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/B, priority=13, startTime=1732762434539; duration=0sec 2024-11-28T02:53:55,027 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:55,027 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:B 2024-11-28T02:53:55,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762495087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762495087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762495087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762495087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,138 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/ca5b2179b73f4925b4da78b6112a5485 2024-11-28T02:53:55,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/6471556c872a4349b2f7712481975e0d is 50, key is test_row_0/C:col10/1732762433599/Put/seqid=0 2024-11-28T02:53:55,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741875_1051 (size=12151) 2024-11-28T02:53:55,186 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/6471556c872a4349b2f7712481975e0d 2024-11-28T02:53:55,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/995ddb1afa1144c0a2d2c072f0d04d0b as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/995ddb1afa1144c0a2d2c072f0d04d0b 2024-11-28T02:53:55,221 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/995ddb1afa1144c0a2d2c072f0d04d0b, entries=150, sequenceid=228, filesize=11.9 K 2024-11-28T02:53:55,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/ca5b2179b73f4925b4da78b6112a5485 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/ca5b2179b73f4925b4da78b6112a5485 2024-11-28T02:53:55,235 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/ca5b2179b73f4925b4da78b6112a5485, entries=150, sequenceid=228, filesize=11.9 K 2024-11-28T02:53:55,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/6471556c872a4349b2f7712481975e0d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/6471556c872a4349b2f7712481975e0d 2024-11-28T02:53:55,260 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/6471556c872a4349b2f7712481975e0d, entries=150, sequenceid=228, filesize=11.9 K 2024-11-28T02:53:55,265 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 599ms, sequenceid=228, compaction requested=false 2024-11-28T02:53:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:55,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-28T02:53:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-28T02:53:55,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-28T02:53:55,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5370 sec 2024-11-28T02:53:55,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.5470 sec 2024-11-28T02:53:55,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T02:53:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:55,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/988a8de0bcf24e8396dcd5ce29c6e6bb is 50, key is test_row_0/A:col10/1732762435398/Put/seqid=0 2024-11-28T02:53:55,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762495458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762495458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762495460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762495461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741876_1052 (size=12151) 2024-11-28T02:53:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762495576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762495577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762495577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762495577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762495787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762495787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762495798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:55,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762495798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:55,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:55,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:55,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/988a8de0bcf24e8396dcd5ce29c6e6bb 2024-11-28T02:53:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T02:53:55,877 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:53:55,879 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:53:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-28T02:53:55,884 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:53:55,886 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:53:55,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:53:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T02:53:55,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/bf29b1b5147943a0ad69fdf835136c95 is 50, key is test_row_0/B:col10/1732762435398/Put/seqid=0 2024-11-28T02:53:55,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741877_1053 (size=12151) 2024-11-28T02:53:55,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/bf29b1b5147943a0ad69fdf835136c95 2024-11-28T02:53:55,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/116e9a733bd64ef5a64d0c1294edb0d5 is 50, key is test_row_0/C:col10/1732762435398/Put/seqid=0 2024-11-28T02:53:55,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741878_1054 (size=12151) 2024-11-28T02:53:55,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/116e9a733bd64ef5a64d0c1294edb0d5 2024-11-28T02:53:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T02:53:56,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/988a8de0bcf24e8396dcd5ce29c6e6bb as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/988a8de0bcf24e8396dcd5ce29c6e6bb 2024-11-28T02:53:56,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/988a8de0bcf24e8396dcd5ce29c6e6bb, entries=150, sequenceid=246, filesize=11.9 K 2024-11-28T02:53:56,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/bf29b1b5147943a0ad69fdf835136c95 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/bf29b1b5147943a0ad69fdf835136c95 2024-11-28T02:53:56,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/bf29b1b5147943a0ad69fdf835136c95, entries=150, sequenceid=246, filesize=11.9 K 2024-11-28T02:53:56,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/116e9a733bd64ef5a64d0c1294edb0d5 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/116e9a733bd64ef5a64d0c1294edb0d5 2024-11-28T02:53:56,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T02:53:56,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:56,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/116e9a733bd64ef5a64d0c1294edb0d5, entries=150, sequenceid=246, filesize=11.9 K 2024-11-28T02:53:56,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 648ms, sequenceid=246, compaction requested=true 2024-11-28T02:53:56,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:56,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:53:56,052 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:53:56,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:56,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:53:56,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:56,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:53:56,052 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:53:56,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:56,053 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:53:56,053 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/B is initiating minor compaction (all files) 2024-11-28T02:53:56,053 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:53:56,054 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/B in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,054 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/A is initiating minor compaction (all files) 2024-11-28T02:53:56,054 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/A in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,054 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/c0d47269dd0f4233b01d52019292d831, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/ca5b2179b73f4925b4da78b6112a5485, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/bf29b1b5147943a0ad69fdf835136c95] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=36.0 K 2024-11-28T02:53:56,054 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/023d301f9755483dab67b4a8ca8b358f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/995ddb1afa1144c0a2d2c072f0d04d0b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/988a8de0bcf24e8396dcd5ce29c6e6bb] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=36.0 K 2024-11-28T02:53:56,054 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d47269dd0f4233b01d52019292d831, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732762432907 2024-11-28T02:53:56,054 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 023d301f9755483dab67b4a8ca8b358f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732762432907 2024-11-28T02:53:56,055 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting ca5b2179b73f4925b4da78b6112a5485, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732762433590 2024-11-28T02:53:56,055 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 995ddb1afa1144c0a2d2c072f0d04d0b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732762433590 2024-11-28T02:53:56,055 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting bf29b1b5147943a0ad69fdf835136c95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732762435398 2024-11-28T02:53:56,056 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 988a8de0bcf24e8396dcd5ce29c6e6bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732762435398 2024-11-28T02:53:56,074 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:56,075 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/6c6e1706b8ae479fbb17c0f056bebda3 is 50, key is test_row_0/B:col10/1732762435398/Put/seqid=0 2024-11-28T02:53:56,085 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:56,086 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/2d79e8df16c24a5ca17efd463f9f1aad is 50, key is test_row_0/A:col10/1732762435398/Put/seqid=0 2024-11-28T02:53:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:56,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T02:53:56,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:56,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:56,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:56,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:56,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:56,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:56,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741879_1055 (size=12629) 2024-11-28T02:53:56,117 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/6c6e1706b8ae479fbb17c0f056bebda3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6c6e1706b8ae479fbb17c0f056bebda3 2024-11-28T02:53:56,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/662f66c626574db7893f1db3cb98eed8 is 50, key is test_row_0/A:col10/1732762435459/Put/seqid=0 2024-11-28T02:53:56,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741880_1056 (size=12629) 2024-11-28T02:53:56,132 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/B of 1fec00bb9c12dd4e6236acd1fe2329f8 into 6c6e1706b8ae479fbb17c0f056bebda3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:56,132 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:56,133 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/B, priority=13, startTime=1732762436052; duration=0sec 2024-11-28T02:53:56,133 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:56,133 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:B 2024-11-28T02:53:56,133 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:53:56,135 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:53:56,135 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/C is initiating minor compaction (all files) 2024-11-28T02:53:56,135 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/C in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,135 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/43658e1ceb674cce9d5cb3c463f0b14e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/6471556c872a4349b2f7712481975e0d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/116e9a733bd64ef5a64d0c1294edb0d5] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=36.0 K 2024-11-28T02:53:56,136 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 43658e1ceb674cce9d5cb3c463f0b14e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732762432907 2024-11-28T02:53:56,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741881_1057 (size=14741) 2024-11-28T02:53:56,140 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6471556c872a4349b2f7712481975e0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732762433590 2024-11-28T02:53:56,142 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 116e9a733bd64ef5a64d0c1294edb0d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732762435398 2024-11-28T02:53:56,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/662f66c626574db7893f1db3cb98eed8 2024-11-28T02:53:56,147 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/2d79e8df16c24a5ca17efd463f9f1aad as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2d79e8df16c24a5ca17efd463f9f1aad 2024-11-28T02:53:56,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762496143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,161 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/A of 1fec00bb9c12dd4e6236acd1fe2329f8 into 2d79e8df16c24a5ca17efd463f9f1aad(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:56,161 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:56,162 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/A, priority=13, startTime=1732762436051; duration=0sec 2024-11-28T02:53:56,162 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:56,162 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:A 2024-11-28T02:53:56,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762496153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/e4b00749c6314a3bb0012157cf236762 is 50, key is test_row_0/B:col10/1732762435459/Put/seqid=0 2024-11-28T02:53:56,173 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#C#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:56,174 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/0fba68832e614e8d8504f88b0c145ca1 is 50, key is test_row_0/C:col10/1732762435398/Put/seqid=0 2024-11-28T02:53:56,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762496158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762496162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T02:53:56,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:56,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T02:53:56,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741882_1058 (size=12301) 2024-11-28T02:53:56,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741883_1059 (size=12629) 2024-11-28T02:53:56,236 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/0fba68832e614e8d8504f88b0c145ca1 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0fba68832e614e8d8504f88b0c145ca1 2024-11-28T02:53:56,246 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/C of 1fec00bb9c12dd4e6236acd1fe2329f8 into 0fba68832e614e8d8504f88b0c145ca1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:56,246 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:56,246 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/C, priority=13, startTime=1732762436052; duration=0sec 2024-11-28T02:53:56,246 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:56,246 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:C 2024-11-28T02:53:56,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762496266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762496276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762496286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762496286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T02:53:56,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:56,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762496476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762496486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762496496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762496496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T02:53:56,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:56,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T02:53:56,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/e4b00749c6314a3bb0012157cf236762 2024-11-28T02:53:56,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/455a40180b2e4766b9be42671bf90cd9 is 50, key is test_row_0/C:col10/1732762435459/Put/seqid=0 2024-11-28T02:53:56,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741884_1060 (size=12301) 2024-11-28T02:53:56,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T02:53:56,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:56,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762496786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762496796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762496806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762496806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:56,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:56,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T02:53:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T02:53:56,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:56,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:56,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T02:53:57,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/455a40180b2e4766b9be42671bf90cd9 2024-11-28T02:53:57,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/662f66c626574db7893f1db3cb98eed8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/662f66c626574db7893f1db3cb98eed8 2024-11-28T02:53:57,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/662f66c626574db7893f1db3cb98eed8, entries=200, sequenceid=264, filesize=14.4 K 2024-11-28T02:53:57,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/e4b00749c6314a3bb0012157cf236762 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/e4b00749c6314a3bb0012157cf236762 2024-11-28T02:53:57,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/e4b00749c6314a3bb0012157cf236762, entries=150, sequenceid=264, filesize=12.0 K 2024-11-28T02:53:57,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/455a40180b2e4766b9be42671bf90cd9 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/455a40180b2e4766b9be42671bf90cd9 2024-11-28T02:53:57,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/455a40180b2e4766b9be42671bf90cd9, entries=150, sequenceid=264, filesize=12.0 K 2024-11-28T02:53:57,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 988ms, sequenceid=264, compaction requested=false 2024-11-28T02:53:57,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:57,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T02:53:57,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:57,134 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T02:53:57,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:57,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:57,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:57,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:57,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:57,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:57,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/4aa36f75120e4383bec659b033d24689 is 50, key is test_row_0/A:col10/1732762436136/Put/seqid=0 2024-11-28T02:53:57,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741885_1061 (size=12301) 2024-11-28T02:53:57,151 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/4aa36f75120e4383bec659b033d24689 2024-11-28T02:53:57,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/21c3fb019a9d4505b47ad759ce4a25b4 is 50, key is test_row_0/B:col10/1732762436136/Put/seqid=0 2024-11-28T02:53:57,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741886_1062 (size=12301) 2024-11-28T02:53:57,180 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/21c3fb019a9d4505b47ad759ce4a25b4 2024-11-28T02:53:57,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/0e05c7129f28415a93f36707db2cca6c is 50, key is test_row_0/C:col10/1732762436136/Put/seqid=0 2024-11-28T02:53:57,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741887_1063 (size=12301) 2024-11-28T02:53:57,223 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/0e05c7129f28415a93f36707db2cca6c 2024-11-28T02:53:57,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/4aa36f75120e4383bec659b033d24689 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/4aa36f75120e4383bec659b033d24689 2024-11-28T02:53:57,246 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/4aa36f75120e4383bec659b033d24689, entries=150, sequenceid=282, filesize=12.0 K 2024-11-28T02:53:57,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/21c3fb019a9d4505b47ad759ce4a25b4 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/21c3fb019a9d4505b47ad759ce4a25b4 2024-11-28T02:53:57,257 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/21c3fb019a9d4505b47ad759ce4a25b4, entries=150, sequenceid=282, filesize=12.0 K 2024-11-28T02:53:57,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/0e05c7129f28415a93f36707db2cca6c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0e05c7129f28415a93f36707db2cca6c 2024-11-28T02:53:57,268 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0e05c7129f28415a93f36707db2cca6c, entries=150, sequenceid=282, filesize=12.0 K 2024-11-28T02:53:57,270 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 136ms, sequenceid=282, compaction requested=true 2024-11-28T02:53:57,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-28T02:53:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-28T02:53:57,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-28T02:53:57,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3880 sec 2024-11-28T02:53:57,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4010 sec 2024-11-28T02:53:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:57,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T02:53:57,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:57,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:57,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:57,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:57,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:57,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:57,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/d51482761c8b4580a7bc5a23fb05878e is 50, key is test_row_0/A:col10/1732762437318/Put/seqid=0 2024-11-28T02:53:57,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741888_1064 (size=14741) 2024-11-28T02:53:57,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762497425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:57,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762497425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:57,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762437422, max timestamp=1732762437426Num uniques 859; 2024-11-28T02:53:57,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:53:57,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762437422, max timestamp=1732762437426Num uniques 859; 2024-11-28T02:53:57,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762437422, max timestamp=1732762437426Num uniques 859; 2024-11-28T02:53:57,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:53:57,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:53:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:57,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762497503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:57,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:57,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762497513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:57,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762497536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:57,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:57,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762497537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:57,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:57,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762497616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:57,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762497626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:57,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762497746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:57,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/d51482761c8b4580a7bc5a23fb05878e 2024-11-28T02:53:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762497746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:57,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/a30ddd2b100f456cb9e2e830bfaa2d30 is 50, key is test_row_0/B:col10/1732762437318/Put/seqid=0 2024-11-28T02:53:57,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741889_1065 (size=12301) 2024-11-28T02:53:57,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762497836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:57,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762497846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:57,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T02:53:58,046 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:53:58,047 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:53:58,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-28T02:53:58,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T02:53:58,056 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:53:58,057 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:53:58,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:53:58,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:58,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762498056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:58,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:58,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762498057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:58,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:58,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762498146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T02:53:58,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:58,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762498156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:58,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/a30ddd2b100f456cb9e2e830bfaa2d30 2024-11-28T02:53:58,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/4e8cfdc765ae45eda98b86edc232d169 is 50, key is test_row_0/C:col10/1732762437318/Put/seqid=0 2024-11-28T02:53:58,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T02:53:58,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:58,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:58,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:58,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:58,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:58,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:58,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741890_1066 (size=12301) 2024-11-28T02:53:58,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/4e8cfdc765ae45eda98b86edc232d169 2024-11-28T02:53:58,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/d51482761c8b4580a7bc5a23fb05878e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d51482761c8b4580a7bc5a23fb05878e 2024-11-28T02:53:58,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d51482761c8b4580a7bc5a23fb05878e, entries=200, sequenceid=293, filesize=14.4 K 2024-11-28T02:53:58,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/a30ddd2b100f456cb9e2e830bfaa2d30 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/a30ddd2b100f456cb9e2e830bfaa2d30 2024-11-28T02:53:58,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/a30ddd2b100f456cb9e2e830bfaa2d30, entries=150, sequenceid=293, filesize=12.0 K 2024-11-28T02:53:58,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/4e8cfdc765ae45eda98b86edc232d169 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4e8cfdc765ae45eda98b86edc232d169 2024-11-28T02:53:58,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4e8cfdc765ae45eda98b86edc232d169, entries=150, sequenceid=293, filesize=12.0 K 2024-11-28T02:53:58,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 988ms, sequenceid=293, compaction requested=true 2024-11-28T02:53:58,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:58,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:53:58,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:58,311 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:53:58,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:53:58,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:58,311 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:53:58,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:53:58,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:58,313 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:53:58,313 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/B is initiating minor compaction (all files) 2024-11-28T02:53:58,313 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/B in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:58,314 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6c6e1706b8ae479fbb17c0f056bebda3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/e4b00749c6314a3bb0012157cf236762, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/21c3fb019a9d4505b47ad759ce4a25b4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/a30ddd2b100f456cb9e2e830bfaa2d30] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=48.4 K 2024-11-28T02:53:58,314 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54412 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:53:58,314 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/A is initiating minor compaction (all files) 2024-11-28T02:53:58,314 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/A in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:58,315 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2d79e8df16c24a5ca17efd463f9f1aad, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/662f66c626574db7893f1db3cb98eed8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/4aa36f75120e4383bec659b033d24689, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d51482761c8b4580a7bc5a23fb05878e] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=53.1 K 2024-11-28T02:53:58,315 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c6e1706b8ae479fbb17c0f056bebda3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732762435398 2024-11-28T02:53:58,315 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d79e8df16c24a5ca17efd463f9f1aad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732762435398 2024-11-28T02:53:58,316 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting e4b00749c6314a3bb0012157cf236762, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732762435457 2024-11-28T02:53:58,318 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 662f66c626574db7893f1db3cb98eed8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732762435454 2024-11-28T02:53:58,318 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 21c3fb019a9d4505b47ad759ce4a25b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732762436136 2024-11-28T02:53:58,318 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aa36f75120e4383bec659b033d24689, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732762436136 2024-11-28T02:53:58,319 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d51482761c8b4580a7bc5a23fb05878e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732762437315 2024-11-28T02:53:58,319 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting a30ddd2b100f456cb9e2e830bfaa2d30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732762437315 2024-11-28T02:53:58,357 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:58,358 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/624416e9a2184636b5a17c3d2b92ba8b is 50, key is test_row_0/B:col10/1732762437318/Put/seqid=0 2024-11-28T02:53:58,365 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#A#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:58,366 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/eeb5c941d4704f04bf90ca424f4a12e7 is 50, key is test_row_0/A:col10/1732762437318/Put/seqid=0 2024-11-28T02:53:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T02:53:58,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T02:53:58,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:58,372 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-28T02:53:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:53:58,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/82c5b84ec2674a73827f40d6c6f91af0 is 50, key is test_row_0/A:col10/1732762437510/Put/seqid=0 2024-11-28T02:53:58,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741891_1067 (size=12915) 2024-11-28T02:53:58,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741892_1068 (size=12915) 2024-11-28T02:53:58,482 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/eeb5c941d4704f04bf90ca424f4a12e7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/eeb5c941d4704f04bf90ca424f4a12e7 2024-11-28T02:53:58,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741893_1069 (size=12301) 2024-11-28T02:53:58,504 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/82c5b84ec2674a73827f40d6c6f91af0 2024-11-28T02:53:58,506 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/A of 1fec00bb9c12dd4e6236acd1fe2329f8 into eeb5c941d4704f04bf90ca424f4a12e7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:58,506 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:58,506 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/A, priority=12, startTime=1732762438311; duration=0sec 2024-11-28T02:53:58,506 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:58,506 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:A 2024-11-28T02:53:58,506 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:53:58,509 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:53:58,509 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/C is initiating minor compaction (all files) 2024-11-28T02:53:58,509 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/C in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:58,509 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0fba68832e614e8d8504f88b0c145ca1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/455a40180b2e4766b9be42671bf90cd9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0e05c7129f28415a93f36707db2cca6c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4e8cfdc765ae45eda98b86edc232d169] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=48.4 K 2024-11-28T02:53:58,511 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fba68832e614e8d8504f88b0c145ca1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732762435398 2024-11-28T02:53:58,512 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 455a40180b2e4766b9be42671bf90cd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732762435457 2024-11-28T02:53:58,513 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e05c7129f28415a93f36707db2cca6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732762436136 2024-11-28T02:53:58,514 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e8cfdc765ae45eda98b86edc232d169, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732762437315 2024-11-28T02:53:58,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/7dba5e142d8d4815b2fe293a03c79859 is 50, key is test_row_0/B:col10/1732762437510/Put/seqid=0 2024-11-28T02:53:58,538 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#C#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:58,540 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/6d08d8c0e28d4987953b5920507c1e5a is 50, key is test_row_0/C:col10/1732762437318/Put/seqid=0 2024-11-28T02:53:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:58,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:58,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741894_1070 (size=12301) 2024-11-28T02:53:58,599 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/7dba5e142d8d4815b2fe293a03c79859 2024-11-28T02:53:58,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762498600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:58,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762498602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:58,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741895_1071 (size=12915) 2024-11-28T02:53:58,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/39000a84580f409e99ad612df4db468a is 50, key is test_row_0/C:col10/1732762437510/Put/seqid=0 2024-11-28T02:53:58,626 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/6d08d8c0e28d4987953b5920507c1e5a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/6d08d8c0e28d4987953b5920507c1e5a 2024-11-28T02:53:58,636 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/C of 1fec00bb9c12dd4e6236acd1fe2329f8 into 6d08d8c0e28d4987953b5920507c1e5a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:58,636 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:58,636 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/C, priority=12, startTime=1732762438311; duration=0sec 2024-11-28T02:53:58,636 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:58,636 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:C 2024-11-28T02:53:58,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741896_1072 (size=12301) 2024-11-28T02:53:58,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762498656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T02:53:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762498676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:58,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762498716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762498716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:58,855 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/624416e9a2184636b5a17c3d2b92ba8b as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/624416e9a2184636b5a17c3d2b92ba8b 2024-11-28T02:53:58,863 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/B of 1fec00bb9c12dd4e6236acd1fe2329f8 into 624416e9a2184636b5a17c3d2b92ba8b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:58,863 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:58,863 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/B, priority=12, startTime=1732762438311; duration=0sec 2024-11-28T02:53:58,863 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:58,863 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:B 2024-11-28T02:53:58,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762498925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:58,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762498926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:58,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:59,058 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/39000a84580f409e99ad612df4db468a 2024-11-28T02:53:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/82c5b84ec2674a73827f40d6c6f91af0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/82c5b84ec2674a73827f40d6c6f91af0 2024-11-28T02:53:59,085 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/82c5b84ec2674a73827f40d6c6f91af0, entries=150, sequenceid=329, filesize=12.0 K 2024-11-28T02:53:59,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/7dba5e142d8d4815b2fe293a03c79859 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/7dba5e142d8d4815b2fe293a03c79859 2024-11-28T02:53:59,101 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/7dba5e142d8d4815b2fe293a03c79859, entries=150, sequenceid=329, filesize=12.0 K 2024-11-28T02:53:59,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/39000a84580f409e99ad612df4db468a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/39000a84580f409e99ad612df4db468a 2024-11-28T02:53:59,117 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/39000a84580f409e99ad612df4db468a, entries=150, sequenceid=329, filesize=12.0 K 2024-11-28T02:53:59,120 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 747ms, sequenceid=329, compaction requested=false 2024-11-28T02:53:59,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:59,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:59,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-28T02:53:59,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-28T02:53:59,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-28T02:53:59,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0640 sec 2024-11-28T02:53:59,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0810 sec 2024-11-28T02:53:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T02:53:59,186 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:53:59,187 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:53:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-28T02:53:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T02:53:59,191 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:53:59,194 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:53:59,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:53:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:53:59,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T02:53:59,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:59,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:59,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:59,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:59,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:59,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:59,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/6e54770c211b4f77b53a9902ca8c6cf8 is 50, key is test_row_0/A:col10/1732762439237/Put/seqid=0 2024-11-28T02:53:59,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741897_1073 (size=14741) 2024-11-28T02:53:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T02:53:59,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T02:53:59,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:59,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:59,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:59,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:59,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:59,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762499369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:59,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762499374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:59,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:59,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762499486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:59,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762499486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T02:53:59,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T02:53:59,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:59,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:59,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:59,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:59,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:59,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T02:53:59,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:59,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:59,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:59,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:59,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:59,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762499676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:59,682 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 1fec00bb9c12dd4e6236acd1fe2329f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:53:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:59,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/6e54770c211b4f77b53a9902ca8c6cf8 2024-11-28T02:53:59,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:59,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762499697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:59,706 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 1fec00bb9c12dd4e6236acd1fe2329f8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:53:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:59,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762499708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:59,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/d784d91dd4ec494b9b47e1f241ef62da is 50, key is test_row_0/B:col10/1732762439237/Put/seqid=0 2024-11-28T02:53:59,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:59,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:53:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762499707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:53:59,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:53:59,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:53:59,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741898_1074 (size=12301) 2024-11-28T02:53:59,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/d784d91dd4ec494b9b47e1f241ef62da 2024-11-28T02:53:59,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/067fde09fb87472cbc9e1dbb127590d7 is 50, key is test_row_0/C:col10/1732762439237/Put/seqid=0 2024-11-28T02:53:59,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741899_1075 (size=12301) 2024-11-28T02:53:59,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/067fde09fb87472cbc9e1dbb127590d7 2024-11-28T02:53:59,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/6e54770c211b4f77b53a9902ca8c6cf8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6e54770c211b4f77b53a9902ca8c6cf8 2024-11-28T02:53:59,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6e54770c211b4f77b53a9902ca8c6cf8, entries=200, sequenceid=345, filesize=14.4 K 2024-11-28T02:53:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T02:53:59,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/d784d91dd4ec494b9b47e1f241ef62da as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/d784d91dd4ec494b9b47e1f241ef62da 2024-11-28T02:53:59,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T02:53:59,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:59,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:53:59,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:59,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:59,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:53:59,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/d784d91dd4ec494b9b47e1f241ef62da, entries=150, sequenceid=345, filesize=12.0 K 2024-11-28T02:53:59,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/067fde09fb87472cbc9e1dbb127590d7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/067fde09fb87472cbc9e1dbb127590d7 2024-11-28T02:53:59,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/067fde09fb87472cbc9e1dbb127590d7, entries=150, sequenceid=345, filesize=12.0 K 2024-11-28T02:53:59,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 603ms, sequenceid=345, compaction requested=true 2024-11-28T02:53:59,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:59,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:53:59,847 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:53:59,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:59,847 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:53:59,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:53:59,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:53:59,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:53:59,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:59,848 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:53:59,849 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/B is initiating minor compaction (all files) 2024-11-28T02:53:59,849 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/B in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:59,849 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/624416e9a2184636b5a17c3d2b92ba8b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/7dba5e142d8d4815b2fe293a03c79859, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/d784d91dd4ec494b9b47e1f241ef62da] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=36.6 K 2024-11-28T02:53:59,850 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 624416e9a2184636b5a17c3d2b92ba8b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732762437315 2024-11-28T02:53:59,851 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:53:59,851 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dba5e142d8d4815b2fe293a03c79859, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732762437490 2024-11-28T02:53:59,851 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/A is initiating minor compaction (all files) 2024-11-28T02:53:59,851 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/A in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:59,852 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/eeb5c941d4704f04bf90ca424f4a12e7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/82c5b84ec2674a73827f40d6c6f91af0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6e54770c211b4f77b53a9902ca8c6cf8] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=39.0 K 2024-11-28T02:53:59,852 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting d784d91dd4ec494b9b47e1f241ef62da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732762438589 2024-11-28T02:53:59,852 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeb5c941d4704f04bf90ca424f4a12e7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732762437315 2024-11-28T02:53:59,854 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82c5b84ec2674a73827f40d6c6f91af0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732762437490 2024-11-28T02:53:59,854 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e54770c211b4f77b53a9902ca8c6cf8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732762438589 2024-11-28T02:53:59,867 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:59,868 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/f8fef764602c4193b8261bd63b99b85e is 50, key is test_row_0/B:col10/1732762439237/Put/seqid=0 2024-11-28T02:53:59,871 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#A#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:59,871 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/43ae2035b337458380ff41b6289ceaf6 is 50, key is test_row_0/A:col10/1732762439237/Put/seqid=0 2024-11-28T02:53:59,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741901_1077 (size=13017) 2024-11-28T02:53:59,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741900_1076 (size=13017) 2024-11-28T02:53:59,898 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/43ae2035b337458380ff41b6289ceaf6 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/43ae2035b337458380ff41b6289ceaf6 2024-11-28T02:53:59,912 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/A of 1fec00bb9c12dd4e6236acd1fe2329f8 into 43ae2035b337458380ff41b6289ceaf6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:53:59,912 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:53:59,912 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/A, priority=13, startTime=1732762439847; duration=0sec 2024-11-28T02:53:59,913 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:53:59,913 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:A 2024-11-28T02:53:59,913 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:53:59,915 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:53:59,915 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/C is initiating minor compaction (all files) 2024-11-28T02:53:59,915 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/C in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:59,915 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/6d08d8c0e28d4987953b5920507c1e5a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/39000a84580f409e99ad612df4db468a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/067fde09fb87472cbc9e1dbb127590d7] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=36.6 K 2024-11-28T02:53:59,916 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d08d8c0e28d4987953b5920507c1e5a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732762437315 2024-11-28T02:53:59,917 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39000a84580f409e99ad612df4db468a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732762437490 2024-11-28T02:53:59,917 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 067fde09fb87472cbc9e1dbb127590d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732762438589 2024-11-28T02:53:59,944 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#C#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:53:59,945 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/417bed8185a34006b97f912de1f6a02c is 50, key is test_row_0/C:col10/1732762439237/Put/seqid=0 2024-11-28T02:53:59,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741902_1078 (size=13017) 2024-11-28T02:53:59,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T02:53:59,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:53:59,978 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T02:53:59,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:53:59,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:59,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:53:59,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:59,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:53:59,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:53:59,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/25ca6e11df974b82a6a7d7ab1e7215bb is 50, key is test_row_0/A:col10/1732762439361/Put/seqid=0 2024-11-28T02:53:59,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741903_1079 (size=12301) 2024-11-28T02:53:59,993 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/25ca6e11df974b82a6a7d7ab1e7215bb 2024-11-28T02:54:00,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/8b7d518ccbeb4c3c82c1e820e6090c5f is 50, key is test_row_0/B:col10/1732762439361/Put/seqid=0 2024-11-28T02:54:00,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:00,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741904_1080 (size=12301) 2024-11-28T02:54:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:54:00,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762500067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:00,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762500068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:00,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762500178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762500178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:00,301 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/f8fef764602c4193b8261bd63b99b85e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/f8fef764602c4193b8261bd63b99b85e 2024-11-28T02:54:00,322 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/B of 1fec00bb9c12dd4e6236acd1fe2329f8 into f8fef764602c4193b8261bd63b99b85e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:00,322 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:00,322 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/B, priority=13, startTime=1732762439847; duration=0sec 2024-11-28T02:54:00,323 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:00,323 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:B 2024-11-28T02:54:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T02:54:00,381 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/417bed8185a34006b97f912de1f6a02c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/417bed8185a34006b97f912de1f6a02c 2024-11-28T02:54:00,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762500386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:00,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:00,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762500387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:00,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:00,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:00,394 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/C of 1fec00bb9c12dd4e6236acd1fe2329f8 into 417bed8185a34006b97f912de1f6a02c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:00,394 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:00,394 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/C, priority=13, startTime=1732762439848; duration=0sec 2024-11-28T02:54:00,394 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:00,394 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:C 2024-11-28T02:54:00,432 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/8b7d518ccbeb4c3c82c1e820e6090c5f 2024-11-28T02:54:00,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/0a432439bdc6411eb193da44e23b32b0 is 50, key is test_row_0/C:col10/1732762439361/Put/seqid=0 2024-11-28T02:54:00,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741905_1081 (size=12301) 2024-11-28T02:54:00,471 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/0a432439bdc6411eb193da44e23b32b0 2024-11-28T02:54:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/25ca6e11df974b82a6a7d7ab1e7215bb as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/25ca6e11df974b82a6a7d7ab1e7215bb 2024-11-28T02:54:00,493 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/25ca6e11df974b82a6a7d7ab1e7215bb, entries=150, sequenceid=365, filesize=12.0 K 2024-11-28T02:54:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/8b7d518ccbeb4c3c82c1e820e6090c5f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/8b7d518ccbeb4c3c82c1e820e6090c5f 2024-11-28T02:54:00,510 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/8b7d518ccbeb4c3c82c1e820e6090c5f, entries=150, sequenceid=365, filesize=12.0 K 2024-11-28T02:54:00,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/0a432439bdc6411eb193da44e23b32b0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0a432439bdc6411eb193da44e23b32b0 2024-11-28T02:54:00,536 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0a432439bdc6411eb193da44e23b32b0, entries=150, sequenceid=365, filesize=12.0 K 2024-11-28T02:54:00,541 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 563ms, sequenceid=365, compaction requested=false 2024-11-28T02:54:00,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:00,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:00,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-28T02:54:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-28T02:54:00,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-28T02:54:00,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3500 sec 2024-11-28T02:54:00,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.3610 sec 2024-11-28T02:54:00,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T02:54:00,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:54:00,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:00,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:54:00,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:00,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:54:00,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:54:00,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/dde6521a918848f7b501da115d242d7e is 50, key is test_row_0/A:col10/1732762440697/Put/seqid=0 2024-11-28T02:54:00,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741906_1082 (size=12301) 2024-11-28T02:54:00,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762500744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:00,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:00,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762500752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:00,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762500856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:00,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762500867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:01,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762501066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:01,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762501076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:01,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/dde6521a918848f7b501da115d242d7e 2024-11-28T02:54:01,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/2b3242b13ab643bfbe0a369a979b505c is 50, key is test_row_0/B:col10/1732762440697/Put/seqid=0 2024-11-28T02:54:01,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741907_1083 (size=12301) 2024-11-28T02:54:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T02:54:01,336 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:01,337 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-28T02:54:01,339 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T02:54:01,340 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:01,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:01,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762501376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:01,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762501386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T02:54:01,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T02:54:01,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:01,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:01,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:01,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:01,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:01,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/2b3242b13ab643bfbe0a369a979b505c 2024-11-28T02:54:01,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/216dc20916bb4b88912138f81ae8f9db is 50, key is test_row_0/C:col10/1732762440697/Put/seqid=0 2024-11-28T02:54:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741908_1084 (size=12301) 2024-11-28T02:54:01,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/216dc20916bb4b88912138f81ae8f9db 2024-11-28T02:54:01,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/dde6521a918848f7b501da115d242d7e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/dde6521a918848f7b501da115d242d7e 2024-11-28T02:54:01,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/dde6521a918848f7b501da115d242d7e, entries=150, sequenceid=382, filesize=12.0 K 2024-11-28T02:54:01,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/2b3242b13ab643bfbe0a369a979b505c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/2b3242b13ab643bfbe0a369a979b505c 2024-11-28T02:54:01,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/2b3242b13ab643bfbe0a369a979b505c, entries=150, sequenceid=382, filesize=12.0 K 2024-11-28T02:54:01,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/216dc20916bb4b88912138f81ae8f9db as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/216dc20916bb4b88912138f81ae8f9db 2024-11-28T02:54:01,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T02:54:01,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:01,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:01,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:01,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:01,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T02:54:01,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/216dc20916bb4b88912138f81ae8f9db, entries=150, sequenceid=382, filesize=12.0 K 2024-11-28T02:54:01,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 964ms, sequenceid=382, compaction requested=true 2024-11-28T02:54:01,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:01,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:01,664 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:01,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:01,664 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:01,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:01,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:01,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:01,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:01,668 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:01,669 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/A is initiating minor compaction (all files) 2024-11-28T02:54:01,669 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/A in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:01,669 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/43ae2035b337458380ff41b6289ceaf6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/25ca6e11df974b82a6a7d7ab1e7215bb, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/dde6521a918848f7b501da115d242d7e] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=36.7 K 2024-11-28T02:54:01,669 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:01,669 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/B is initiating minor compaction (all files) 2024-11-28T02:54:01,669 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/B in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:01,670 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/f8fef764602c4193b8261bd63b99b85e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/8b7d518ccbeb4c3c82c1e820e6090c5f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/2b3242b13ab643bfbe0a369a979b505c] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=36.7 K 2024-11-28T02:54:01,670 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43ae2035b337458380ff41b6289ceaf6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732762438589 2024-11-28T02:54:01,670 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting f8fef764602c4193b8261bd63b99b85e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732762438589 2024-11-28T02:54:01,671 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25ca6e11df974b82a6a7d7ab1e7215bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732762439361 2024-11-28T02:54:01,671 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b7d518ccbeb4c3c82c1e820e6090c5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732762439361 2024-11-28T02:54:01,671 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b3242b13ab643bfbe0a369a979b505c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732762440057 2024-11-28T02:54:01,671 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting dde6521a918848f7b501da115d242d7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732762440057 2024-11-28T02:54:01,693 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#B#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:01,694 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/b201e2bf134e40dc835a22ba399246bd is 50, key is test_row_0/B:col10/1732762440697/Put/seqid=0 2024-11-28T02:54:01,717 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#A#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:01,718 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/ac406e3aba754128a34fa9657085b12c is 50, key is test_row_0/A:col10/1732762440697/Put/seqid=0 2024-11-28T02:54:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:54:01,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T02:54:01,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:54:01,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:01,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:54:01,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:01,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:54:01,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:01,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/0ec39ad096364b53aaa457bb639cd9c3 is 50, key is test_row_0/A:col10/1732762440726/Put/seqid=0 2024-11-28T02:54:01,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741909_1085 (size=13119) 2024-11-28T02:54:01,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:01,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762501777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:01,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762501785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:01,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T02:54:01,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:01,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:01,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:01,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:01,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:01,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741910_1086 (size=13119) 2024-11-28T02:54:01,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741911_1087 (size=12301) 2024-11-28T02:54:01,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/0ec39ad096364b53aaa457bb639cd9c3 2024-11-28T02:54:01,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/71e470a1510a49aa82ede15ed2853f7a is 50, key is test_row_0/B:col10/1732762440726/Put/seqid=0 2024-11-28T02:54:01,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762501886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:01,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:01,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741912_1088 (size=12301) 2024-11-28T02:54:01,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762501896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762501896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:01,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762501916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:01,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T02:54:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:01,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:01,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:01,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:01,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T02:54:02,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762502106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:02,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T02:54:02,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:02,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:02,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:02,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762502126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:02,193 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/b201e2bf134e40dc835a22ba399246bd as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b201e2bf134e40dc835a22ba399246bd 2024-11-28T02:54:02,203 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/B of 1fec00bb9c12dd4e6236acd1fe2329f8 into b201e2bf134e40dc835a22ba399246bd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:02,203 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:02,203 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/B, priority=13, startTime=1732762441664; duration=0sec 2024-11-28T02:54:02,203 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:02,203 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:B 2024-11-28T02:54:02,203 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:02,205 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:02,205 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/C is initiating minor compaction (all files) 2024-11-28T02:54:02,205 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/C in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:02,205 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/417bed8185a34006b97f912de1f6a02c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0a432439bdc6411eb193da44e23b32b0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/216dc20916bb4b88912138f81ae8f9db] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=36.7 K 2024-11-28T02:54:02,206 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 417bed8185a34006b97f912de1f6a02c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732762438589 2024-11-28T02:54:02,206 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a432439bdc6411eb193da44e23b32b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732762439361 2024-11-28T02:54:02,206 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 216dc20916bb4b88912138f81ae8f9db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732762440057 2024-11-28T02:54:02,215 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#C#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:02,216 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/a158045b656443bbbd3ab251ce83b417 is 50, key is test_row_0/C:col10/1732762440697/Put/seqid=0 2024-11-28T02:54:02,221 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/ac406e3aba754128a34fa9657085b12c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/ac406e3aba754128a34fa9657085b12c 2024-11-28T02:54:02,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741913_1089 (size=13119) 2024-11-28T02:54:02,228 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/A of 1fec00bb9c12dd4e6236acd1fe2329f8 into ac406e3aba754128a34fa9657085b12c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:02,228 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:02,228 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/A, priority=13, startTime=1732762441664; duration=0sec 2024-11-28T02:54:02,228 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:02,228 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:A 2024-11-28T02:54:02,234 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/a158045b656443bbbd3ab251ce83b417 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/a158045b656443bbbd3ab251ce83b417 2024-11-28T02:54:02,240 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/C of 1fec00bb9c12dd4e6236acd1fe2329f8 into a158045b656443bbbd3ab251ce83b417(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:02,240 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:02,240 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/C, priority=13, startTime=1732762441664; duration=0sec 2024-11-28T02:54:02,240 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:02,241 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:C 2024-11-28T02:54:02,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T02:54:02,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:02,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:02,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:02,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/71e470a1510a49aa82ede15ed2853f7a 2024-11-28T02:54:02,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/27582b36dd934cd69bc0b480a202d3af is 50, key is test_row_0/C:col10/1732762440726/Put/seqid=0 2024-11-28T02:54:02,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741914_1090 (size=12301) 2024-11-28T02:54:02,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762502416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:02,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T02:54:02,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:02,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:02,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:02,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762502435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T02:54:02,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T02:54:02,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:02,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:02,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:02,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/27582b36dd934cd69bc0b480a202d3af 2024-11-28T02:54:02,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/0ec39ad096364b53aaa457bb639cd9c3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/0ec39ad096364b53aaa457bb639cd9c3 2024-11-28T02:54:02,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T02:54:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:02,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:02,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/0ec39ad096364b53aaa457bb639cd9c3, entries=150, sequenceid=400, filesize=12.0 K 2024-11-28T02:54:02,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/71e470a1510a49aa82ede15ed2853f7a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/71e470a1510a49aa82ede15ed2853f7a 2024-11-28T02:54:02,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/71e470a1510a49aa82ede15ed2853f7a, entries=150, sequenceid=400, filesize=12.0 K 2024-11-28T02:54:02,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/27582b36dd934cd69bc0b480a202d3af as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/27582b36dd934cd69bc0b480a202d3af 2024-11-28T02:54:02,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/27582b36dd934cd69bc0b480a202d3af, entries=150, sequenceid=400, filesize=12.0 K 2024-11-28T02:54:02,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 1028ms, sequenceid=400, compaction requested=false 2024-11-28T02:54:02,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:02,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T02:54:02,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:02,891 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T02:54:02,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:54:02,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:02,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:54:02,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:02,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:54:02,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:02,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:54:02,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/d815be114d3d419a99af1327e88bee09 is 50, key is test_row_0/A:col10/1732762441765/Put/seqid=0 2024-11-28T02:54:02,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741915_1091 (size=12301) 2024-11-28T02:54:02,933 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/d815be114d3d419a99af1327e88bee09 2024-11-28T02:54:02,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/b83b8c348e5b450f8d13ba380dfced9a is 50, key is test_row_0/B:col10/1732762441765/Put/seqid=0 2024-11-28T02:54:02,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762502953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:02,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762502955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:02,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762502956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:02,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741916_1092 (size=12301) 2024-11-28T02:54:02,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762502959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:02,969 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/b83b8c348e5b450f8d13ba380dfced9a 2024-11-28T02:54:02,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:02,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:02,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/b2f8d7190e39435cbb7df10505aec5ea is 50, key is test_row_0/C:col10/1732762441765/Put/seqid=0 2024-11-28T02:54:02,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741917_1093 (size=12301) 2024-11-28T02:54:03,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762503066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762503066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762503083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762503276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762503276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762503296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,408 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/b2f8d7190e39435cbb7df10505aec5ea 2024-11-28T02:54:03,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/d815be114d3d419a99af1327e88bee09 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d815be114d3d419a99af1327e88bee09 2024-11-28T02:54:03,435 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d815be114d3d419a99af1327e88bee09, entries=150, sequenceid=417, filesize=12.0 K 2024-11-28T02:54:03,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/b83b8c348e5b450f8d13ba380dfced9a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b83b8c348e5b450f8d13ba380dfced9a 2024-11-28T02:54:03,454 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b83b8c348e5b450f8d13ba380dfced9a, entries=150, sequenceid=417, filesize=12.0 K 2024-11-28T02:54:03,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/b2f8d7190e39435cbb7df10505aec5ea as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b2f8d7190e39435cbb7df10505aec5ea 2024-11-28T02:54:03,468 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b2f8d7190e39435cbb7df10505aec5ea, entries=150, sequenceid=417, filesize=12.0 K 2024-11-28T02:54:03,470 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 579ms, sequenceid=417, compaction requested=true 2024-11-28T02:54:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-28T02:54:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-28T02:54:03,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-28T02:54:03,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1330 sec 2024-11-28T02:54:03,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.1430 sec 2024-11-28T02:54:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T02:54:03,486 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:03,487 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-28T02:54:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T02:54:03,490 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:03,491 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:03,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:54:03,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T02:54:03,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:54:03,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:03,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:54:03,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:03,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:54:03,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:03,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/0bcf206322694e0ca93e9d9ea4f9be48 is 50, key is test_row_0/A:col10/1732762442948/Put/seqid=0 2024-11-28T02:54:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T02:54:03,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762503630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762503631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762503633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T02:54:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:03,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:03,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:03,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741918_1094 (size=14741) 2024-11-28T02:54:03,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/0bcf206322694e0ca93e9d9ea4f9be48 2024-11-28T02:54:03,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/6830820587a24708b4399cd02e42abbc is 50, key is test_row_0/B:col10/1732762442948/Put/seqid=0 2024-11-28T02:54:03,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741919_1095 (size=12301) 2024-11-28T02:54:03,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/6830820587a24708b4399cd02e42abbc 2024-11-28T02:54:03,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/84dd794e4b7446be963b1252494cfd8c is 50, key is test_row_0/C:col10/1732762442948/Put/seqid=0 2024-11-28T02:54:03,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762503746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762503746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762503746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741920_1096 (size=12301) 2024-11-28T02:54:03,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/84dd794e4b7446be963b1252494cfd8c 2024-11-28T02:54:03,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/0bcf206322694e0ca93e9d9ea4f9be48 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/0bcf206322694e0ca93e9d9ea4f9be48 2024-11-28T02:54:03,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/0bcf206322694e0ca93e9d9ea4f9be48, entries=200, sequenceid=436, filesize=14.4 K 2024-11-28T02:54:03,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/6830820587a24708b4399cd02e42abbc as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6830820587a24708b4399cd02e42abbc 2024-11-28T02:54:03,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T02:54:03,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:03,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:03,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:03,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:03,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T02:54:03,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6830820587a24708b4399cd02e42abbc, entries=150, sequenceid=436, filesize=12.0 K 2024-11-28T02:54:03,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/84dd794e4b7446be963b1252494cfd8c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/84dd794e4b7446be963b1252494cfd8c 2024-11-28T02:54:03,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/84dd794e4b7446be963b1252494cfd8c, entries=150, sequenceid=436, filesize=12.0 K 2024-11-28T02:54:03,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 248ms, sequenceid=436, compaction requested=true 2024-11-28T02:54:03,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:03,837 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:54:03,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:03,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:03,838 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:54:03,840 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:54:03,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:03,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:03,840 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/B is initiating minor compaction (all files) 2024-11-28T02:54:03,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:03,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:03,840 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/B in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:03,841 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b201e2bf134e40dc835a22ba399246bd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/71e470a1510a49aa82ede15ed2853f7a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b83b8c348e5b450f8d13ba380dfced9a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6830820587a24708b4399cd02e42abbc] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=48.8 K 2024-11-28T02:54:03,842 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52462 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:54:03,842 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting b201e2bf134e40dc835a22ba399246bd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732762440057 2024-11-28T02:54:03,843 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/A is initiating minor compaction (all files) 2024-11-28T02:54:03,843 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/A in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:03,843 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/ac406e3aba754128a34fa9657085b12c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/0ec39ad096364b53aaa457bb639cd9c3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d815be114d3d419a99af1327e88bee09, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/0bcf206322694e0ca93e9d9ea4f9be48] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=51.2 K 2024-11-28T02:54:03,844 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 71e470a1510a49aa82ede15ed2853f7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732762440726 2024-11-28T02:54:03,844 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac406e3aba754128a34fa9657085b12c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732762440057 2024-11-28T02:54:03,845 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting b83b8c348e5b450f8d13ba380dfced9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732762441765 2024-11-28T02:54:03,845 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ec39ad096364b53aaa457bb639cd9c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732762440726 2024-11-28T02:54:03,845 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6830820587a24708b4399cd02e42abbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732762442947 2024-11-28T02:54:03,847 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d815be114d3d419a99af1327e88bee09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732762441765 2024-11-28T02:54:03,848 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bcf206322694e0ca93e9d9ea4f9be48, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732762442947 2024-11-28T02:54:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:54:03,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T02:54:03,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:54:03,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:03,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:54:03,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:03,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:54:03,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:03,883 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#B#compaction#84 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:03,884 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/f1428bd9b9444bc683b9b9129fa5cf4c is 50, key is test_row_0/B:col10/1732762442948/Put/seqid=0 2024-11-28T02:54:03,896 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#A#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:03,897 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/dc596c26590e40c396f3306c1254b4f0 is 50, key is test_row_0/A:col10/1732762442948/Put/seqid=0 2024-11-28T02:54:03,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741921_1097 (size=13255) 2024-11-28T02:54:03,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/2803b62bdcf44e2194ea09acd9c3bd7f is 50, key is test_row_0/A:col10/1732762443868/Put/seqid=0 2024-11-28T02:54:03,936 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/f1428bd9b9444bc683b9b9129fa5cf4c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/f1428bd9b9444bc683b9b9129fa5cf4c 2024-11-28T02:54:03,944 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/B of 1fec00bb9c12dd4e6236acd1fe2329f8 into f1428bd9b9444bc683b9b9129fa5cf4c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:03,944 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:03,944 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/B, priority=12, startTime=1732762443838; duration=0sec 2024-11-28T02:54:03,945 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:03,945 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:B 2024-11-28T02:54:03,945 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:54:03,947 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:54:03,948 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 1fec00bb9c12dd4e6236acd1fe2329f8/C is initiating minor compaction (all files) 2024-11-28T02:54:03,948 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1fec00bb9c12dd4e6236acd1fe2329f8/C in TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:03,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741922_1098 (size=13255) 2024-11-28T02:54:03,948 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/a158045b656443bbbd3ab251ce83b417, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/27582b36dd934cd69bc0b480a202d3af, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b2f8d7190e39435cbb7df10505aec5ea, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/84dd794e4b7446be963b1252494cfd8c] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp, totalSize=48.8 K 2024-11-28T02:54:03,949 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting a158045b656443bbbd3ab251ce83b417, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1732762440057 2024-11-28T02:54:03,950 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 27582b36dd934cd69bc0b480a202d3af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732762440726 2024-11-28T02:54:03,951 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f8d7190e39435cbb7df10505aec5ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732762441765 2024-11-28T02:54:03,952 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 84dd794e4b7446be963b1252494cfd8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732762442947 2024-11-28T02:54:03,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T02:54:03,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:03,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:03,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:03,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:03,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:03,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:03,959 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/dc596c26590e40c396f3306c1254b4f0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/dc596c26590e40c396f3306c1254b4f0 2024-11-28T02:54:03,969 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/A of 1fec00bb9c12dd4e6236acd1fe2329f8 into dc596c26590e40c396f3306c1254b4f0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:03,969 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:03,969 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/A, priority=12, startTime=1732762443837; duration=0sec 2024-11-28T02:54:03,969 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:03,970 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:A 2024-11-28T02:54:03,977 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1fec00bb9c12dd4e6236acd1fe2329f8#C#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:03,978 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/d8f91f853abb446f80fb57007373ec5e is 50, key is test_row_0/C:col10/1732762442948/Put/seqid=0 2024-11-28T02:54:03,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741923_1099 (size=14741) 2024-11-28T02:54:03,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/2803b62bdcf44e2194ea09acd9c3bd7f 2024-11-28T02:54:03,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762503978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762503978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762503986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762503995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:03,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762503995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741924_1100 (size=13255) 2024-11-28T02:54:04,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/b004e307ee3c4ef1bf4e144d1a233843 is 50, key is test_row_0/B:col10/1732762443868/Put/seqid=0 2024-11-28T02:54:04,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741925_1101 (size=12301) 2024-11-28T02:54:04,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/b004e307ee3c4ef1bf4e144d1a233843 2024-11-28T02:54:04,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/322b7fae8ff34a71975b3e1ecd81d9ec is 50, key is test_row_0/C:col10/1732762443868/Put/seqid=0 2024-11-28T02:54:04,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741926_1102 (size=12301) 2024-11-28T02:54:04,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762504106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T02:54:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762504106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:04,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762504107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:04,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762504107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762504107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T02:54:04,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T02:54:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:04,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:04,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762504317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762504317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762504317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762504318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762504318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T02:54:04,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:04,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:04,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:04,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:04,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:04,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:04,424 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/d8f91f853abb446f80fb57007373ec5e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/d8f91f853abb446f80fb57007373ec5e 2024-11-28T02:54:04,438 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1fec00bb9c12dd4e6236acd1fe2329f8/C of 1fec00bb9c12dd4e6236acd1fe2329f8 into d8f91f853abb446f80fb57007373ec5e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:04,438 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:04,438 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., storeName=1fec00bb9c12dd4e6236acd1fe2329f8/C, priority=12, startTime=1732762443840; duration=0sec 2024-11-28T02:54:04,438 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:04,438 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:C 2024-11-28T02:54:04,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/322b7fae8ff34a71975b3e1ecd81d9ec 2024-11-28T02:54:04,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/2803b62bdcf44e2194ea09acd9c3bd7f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2803b62bdcf44e2194ea09acd9c3bd7f 2024-11-28T02:54:04,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2803b62bdcf44e2194ea09acd9c3bd7f, entries=200, sequenceid=450, filesize=14.4 K 2024-11-28T02:54:04,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/b004e307ee3c4ef1bf4e144d1a233843 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b004e307ee3c4ef1bf4e144d1a233843 2024-11-28T02:54:04,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b004e307ee3c4ef1bf4e144d1a233843, entries=150, sequenceid=450, filesize=12.0 K 2024-11-28T02:54:04,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/322b7fae8ff34a71975b3e1ecd81d9ec as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/322b7fae8ff34a71975b3e1ecd81d9ec 2024-11-28T02:54:04,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/322b7fae8ff34a71975b3e1ecd81d9ec, entries=150, sequenceid=450, filesize=12.0 K 2024-11-28T02:54:04,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 621ms, sequenceid=450, compaction requested=false 2024-11-28T02:54:04,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:04,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T02:54:04,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:04,577 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T02:54:04,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:54:04,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:04,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:54:04,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:04,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:54:04,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:04,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/bdec06cde9c84f08a37ef6047c6243d3 is 50, key is test_row_0/A:col10/1732762443973/Put/seqid=0 2024-11-28T02:54:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T02:54:04,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. as already flushing 2024-11-28T02:54:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:54:04,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741927_1103 (size=12301) 2024-11-28T02:54:04,649 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/bdec06cde9c84f08a37ef6047c6243d3 2024-11-28T02:54:04,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762504651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762504651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762504659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762504659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/5a73998ee7654613aa3b268aedd33e88 is 50, key is test_row_0/B:col10/1732762443973/Put/seqid=0 2024-11-28T02:54:04,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762504664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741928_1104 (size=12301) 2024-11-28T02:54:04,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762504767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762504776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762504777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762504777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762504777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762504976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762504996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762504997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762504997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762504997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:05,086 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/5a73998ee7654613aa3b268aedd33e88 2024-11-28T02:54:05,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/73b20fa27b064105b29def03c1f18d1d is 50, key is test_row_0/C:col10/1732762443973/Put/seqid=0 2024-11-28T02:54:05,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741929_1105 (size=12301) 2024-11-28T02:54:05,250 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-28T02:54:05,250 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:05,253 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-28T02:54:05,253 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:05,256 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,256 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,256 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,257 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:05,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:05,258 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-28T02:54:05,258 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:05,258 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,258 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:05,259 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-28T02:54:05,259 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:05,259 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,259 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:05,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53022 deadline: 1732762505286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:05,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:05,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:05,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53024 deadline: 1732762505306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52988 deadline: 1732762505306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53000 deadline: 1732762505306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52966 deadline: 1732762505306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:05,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:05,505 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/73b20fa27b064105b29def03c1f18d1d 2024-11-28T02:54:05,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/bdec06cde9c84f08a37ef6047c6243d3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/bdec06cde9c84f08a37ef6047c6243d3 2024-11-28T02:54:05,516 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/bdec06cde9c84f08a37ef6047c6243d3, entries=150, sequenceid=471, filesize=12.0 K 2024-11-28T02:54:05,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/5a73998ee7654613aa3b268aedd33e88 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/5a73998ee7654613aa3b268aedd33e88 2024-11-28T02:54:05,522 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/5a73998ee7654613aa3b268aedd33e88, entries=150, sequenceid=471, filesize=12.0 K 2024-11-28T02:54:05,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/73b20fa27b064105b29def03c1f18d1d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/73b20fa27b064105b29def03c1f18d1d 2024-11-28T02:54:05,529 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/73b20fa27b064105b29def03c1f18d1d, entries=150, sequenceid=471, filesize=12.0 K 2024-11-28T02:54:05,530 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 952ms, sequenceid=471, compaction requested=true 2024-11-28T02:54:05,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:05,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:05,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-28T02:54:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-28T02:54:05,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-28T02:54:05,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0400 sec 2024-11-28T02:54:05,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.0460 sec 2024-11-28T02:54:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T02:54:05,637 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:54:05,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T02:54:05,798 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-28T02:54:05,798 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:05,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:54:05,798 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:05,798 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:54:05,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:05,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:05,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:54:05,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:05,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/ebb8b154810a452ca7bb4f7677333962 is 50, key is test_row_0/A:col10/1732762445796/Put/seqid=0 2024-11-28T02:54:05,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741930_1106 (size=12301) 2024-11-28T02:54:05,818 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-28T02:54:05,818 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-28T02:54:05,818 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-28T02:54:05,818 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:05,818 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:05,818 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,818 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:05,819 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,819 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,819 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,819 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,819 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:05,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:05,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:05,820 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-28T02:54:05,820 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:05,820 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,820 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:05,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T02:54:05,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-28T02:54:05,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-28T02:54:05,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-28T02:54:05,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-28T02:54:05,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-28T02:54:05,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T02:54:05,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2162 2024-11-28T02:54:05,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2172 2024-11-28T02:54:05,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T02:54:05,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1008 2024-11-28T02:54:05,821 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3016 rows 2024-11-28T02:54:05,822 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 999 2024-11-28T02:54:05,822 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2986 rows 2024-11-28T02:54:05,822 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:54:05,822 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:54:05,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:05,822 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:05,827 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T02:54:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T02:54:05,834 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762445834"}]},"ts":"1732762445834"} 2024-11-28T02:54:05,836 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T02:54:05,836 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T02:54:05,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T02:54:05,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fec00bb9c12dd4e6236acd1fe2329f8, UNASSIGN}] 2024-11-28T02:54:05,844 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fec00bb9c12dd4e6236acd1fe2329f8, UNASSIGN 2024-11-28T02:54:05,845 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1fec00bb9c12dd4e6236acd1fe2329f8, regionState=CLOSING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:54:05,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fec00bb9c12dd4e6236acd1fe2329f8, UNASSIGN because future has completed 2024-11-28T02:54:05,848 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T02:54:05,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:54:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T02:54:06,006 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:54:06,006 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T02:54:06,007 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 1fec00bb9c12dd4e6236acd1fe2329f8, disabling compactions & flushes 2024-11-28T02:54:06,007 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T02:54:06,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/ebb8b154810a452ca7bb4f7677333962 2024-11-28T02:54:06,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/7be314f9ff1e433fa16415df44338e41 is 50, key is test_row_0/B:col10/1732762445796/Put/seqid=0 2024-11-28T02:54:06,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741931_1107 (size=12301) 2024-11-28T02:54:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T02:54:06,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/7be314f9ff1e433fa16415df44338e41 2024-11-28T02:54:06,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/b503cc3de19947f6964acf95f5e9bf3c is 50, key is test_row_0/C:col10/1732762445796/Put/seqid=0 2024-11-28T02:54:06,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741932_1108 (size=12301) 2024-11-28T02:54:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T02:54:07,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/b503cc3de19947f6964acf95f5e9bf3c 2024-11-28T02:54:07,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/ebb8b154810a452ca7bb4f7677333962 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/ebb8b154810a452ca7bb4f7677333962 2024-11-28T02:54:07,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/ebb8b154810a452ca7bb4f7677333962, entries=150, sequenceid=486, filesize=12.0 K 2024-11-28T02:54:07,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/7be314f9ff1e433fa16415df44338e41 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/7be314f9ff1e433fa16415df44338e41 2024-11-28T02:54:07,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/7be314f9ff1e433fa16415df44338e41, entries=150, sequenceid=486, filesize=12.0 K 2024-11-28T02:54:07,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/b503cc3de19947f6964acf95f5e9bf3c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b503cc3de19947f6964acf95f5e9bf3c 2024-11-28T02:54:07,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b503cc3de19947f6964acf95f5e9bf3c, entries=150, sequenceid=486, filesize=12.0 K 2024-11-28T02:54:07,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=26.84 KB/27480 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 1268ms, sequenceid=486, compaction requested=true 2024-11-28T02:54:07,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1fec00bb9c12dd4e6236acd1fe2329f8: 2024-11-28T02:54:07,066 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:07,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:07,066 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:07,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:07,066 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. after waiting 0 ms 2024-11-28T02:54:07,066 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:07,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:07,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:07,066 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 1fec00bb9c12dd4e6236acd1fe2329f8 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-28T02:54:07,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1fec00bb9c12dd4e6236acd1fe2329f8:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:07,066 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. because compaction request was cancelled 2024-11-28T02:54:07,066 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. because compaction request was cancelled 2024-11-28T02:54:07,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:07,066 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:B 2024-11-28T02:54:07,066 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:A 2024-11-28T02:54:07,066 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. because compaction request was cancelled 2024-11-28T02:54:07,066 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1fec00bb9c12dd4e6236acd1fe2329f8:C 2024-11-28T02:54:07,066 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=A 2024-11-28T02:54:07,066 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:07,066 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=B 2024-11-28T02:54:07,067 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:07,067 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1fec00bb9c12dd4e6236acd1fe2329f8, store=C 2024-11-28T02:54:07,067 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:07,071 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/01520e839f9847f0b4cb9f323f6bd4bf is 50, key is test_row_1/A:col10/1732762445818/Put/seqid=0 2024-11-28T02:54:07,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741933_1109 (size=7415) 2024-11-28T02:54:07,477 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/01520e839f9847f0b4cb9f323f6bd4bf 2024-11-28T02:54:07,487 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/15375fdce6d945f69673f39dcb534dfe is 50, key is test_row_1/B:col10/1732762445818/Put/seqid=0 2024-11-28T02:54:07,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741934_1110 (size=7415) 2024-11-28T02:54:07,893 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/15375fdce6d945f69673f39dcb534dfe 2024-11-28T02:54:07,904 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/2bb3d494ba444db7995bbcf21bcf4461 is 50, key is test_row_1/C:col10/1732762445818/Put/seqid=0 2024-11-28T02:54:07,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741935_1111 (size=7415) 2024-11-28T02:54:07,915 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/2bb3d494ba444db7995bbcf21bcf4461 2024-11-28T02:54:07,924 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/A/01520e839f9847f0b4cb9f323f6bd4bf as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/01520e839f9847f0b4cb9f323f6bd4bf 2024-11-28T02:54:07,930 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/01520e839f9847f0b4cb9f323f6bd4bf, entries=50, sequenceid=493, filesize=7.2 K 2024-11-28T02:54:07,931 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/B/15375fdce6d945f69673f39dcb534dfe as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/15375fdce6d945f69673f39dcb534dfe 2024-11-28T02:54:07,937 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/15375fdce6d945f69673f39dcb534dfe, entries=50, sequenceid=493, filesize=7.2 K 2024-11-28T02:54:07,938 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/.tmp/C/2bb3d494ba444db7995bbcf21bcf4461 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/2bb3d494ba444db7995bbcf21bcf4461 2024-11-28T02:54:07,945 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/2bb3d494ba444db7995bbcf21bcf4461, entries=50, sequenceid=493, filesize=7.2 K 2024-11-28T02:54:07,946 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 880ms, sequenceid=493, compaction requested=true 2024-11-28T02:54:07,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6d55513d0598428fa90b5b9952250792, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d0623743d0f944059e75fc2412c7052c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/5996478ad4604910bc1a9c42cf1f0008, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/180b5fab41c449d78e638e182fbefd7c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6ec6f71b95ce4a14810b9bf71993720b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2e40fa16aa2f415cb4777897ff7c0f75, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/166812658f9a49f69c5b2a736a06a4b4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/4ed297548fe44febaf84807894766b73, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/eae2c35a1e9649318d256cf682a33206, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/3c1b0fbc4d444755a315b4d56cdbafa4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/9802668b56524209924414ad01ed63c0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/023d301f9755483dab67b4a8ca8b358f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/995ddb1afa1144c0a2d2c072f0d04d0b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2d79e8df16c24a5ca17efd463f9f1aad, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/988a8de0bcf24e8396dcd5ce29c6e6bb, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/662f66c626574db7893f1db3cb98eed8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/4aa36f75120e4383bec659b033d24689, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d51482761c8b4580a7bc5a23fb05878e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/eeb5c941d4704f04bf90ca424f4a12e7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/82c5b84ec2674a73827f40d6c6f91af0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6e54770c211b4f77b53a9902ca8c6cf8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/43ae2035b337458380ff41b6289ceaf6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/25ca6e11df974b82a6a7d7ab1e7215bb, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/ac406e3aba754128a34fa9657085b12c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/dde6521a918848f7b501da115d242d7e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/0ec39ad096364b53aaa457bb639cd9c3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d815be114d3d419a99af1327e88bee09, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/0bcf206322694e0ca93e9d9ea4f9be48] to archive 2024-11-28T02:54:07,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:54:07,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6d55513d0598428fa90b5b9952250792 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6d55513d0598428fa90b5b9952250792 2024-11-28T02:54:07,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d0623743d0f944059e75fc2412c7052c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d0623743d0f944059e75fc2412c7052c 2024-11-28T02:54:07,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/5996478ad4604910bc1a9c42cf1f0008 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/5996478ad4604910bc1a9c42cf1f0008 2024-11-28T02:54:07,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/180b5fab41c449d78e638e182fbefd7c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/180b5fab41c449d78e638e182fbefd7c 2024-11-28T02:54:07,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6ec6f71b95ce4a14810b9bf71993720b to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6ec6f71b95ce4a14810b9bf71993720b 2024-11-28T02:54:07,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2e40fa16aa2f415cb4777897ff7c0f75 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2e40fa16aa2f415cb4777897ff7c0f75 2024-11-28T02:54:07,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/166812658f9a49f69c5b2a736a06a4b4 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/166812658f9a49f69c5b2a736a06a4b4 2024-11-28T02:54:07,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/4ed297548fe44febaf84807894766b73 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/4ed297548fe44febaf84807894766b73 2024-11-28T02:54:07,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/eae2c35a1e9649318d256cf682a33206 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/eae2c35a1e9649318d256cf682a33206 2024-11-28T02:54:07,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/3c1b0fbc4d444755a315b4d56cdbafa4 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/3c1b0fbc4d444755a315b4d56cdbafa4 2024-11-28T02:54:07,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/9802668b56524209924414ad01ed63c0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/9802668b56524209924414ad01ed63c0 2024-11-28T02:54:07,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/023d301f9755483dab67b4a8ca8b358f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/023d301f9755483dab67b4a8ca8b358f 2024-11-28T02:54:07,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/995ddb1afa1144c0a2d2c072f0d04d0b to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/995ddb1afa1144c0a2d2c072f0d04d0b 2024-11-28T02:54:07,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2d79e8df16c24a5ca17efd463f9f1aad to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2d79e8df16c24a5ca17efd463f9f1aad 2024-11-28T02:54:07,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/988a8de0bcf24e8396dcd5ce29c6e6bb to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/988a8de0bcf24e8396dcd5ce29c6e6bb 2024-11-28T02:54:07,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/662f66c626574db7893f1db3cb98eed8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/662f66c626574db7893f1db3cb98eed8 2024-11-28T02:54:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T02:54:07,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/4aa36f75120e4383bec659b033d24689 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/4aa36f75120e4383bec659b033d24689 2024-11-28T02:54:07,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d51482761c8b4580a7bc5a23fb05878e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d51482761c8b4580a7bc5a23fb05878e 2024-11-28T02:54:07,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/eeb5c941d4704f04bf90ca424f4a12e7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/eeb5c941d4704f04bf90ca424f4a12e7 2024-11-28T02:54:07,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/82c5b84ec2674a73827f40d6c6f91af0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/82c5b84ec2674a73827f40d6c6f91af0 2024-11-28T02:54:07,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6e54770c211b4f77b53a9902ca8c6cf8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/6e54770c211b4f77b53a9902ca8c6cf8 2024-11-28T02:54:07,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/43ae2035b337458380ff41b6289ceaf6 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/43ae2035b337458380ff41b6289ceaf6 2024-11-28T02:54:07,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/25ca6e11df974b82a6a7d7ab1e7215bb to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/25ca6e11df974b82a6a7d7ab1e7215bb 2024-11-28T02:54:07,996 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T02:54:07,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/ac406e3aba754128a34fa9657085b12c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/ac406e3aba754128a34fa9657085b12c 2024-11-28T02:54:08,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/dde6521a918848f7b501da115d242d7e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/dde6521a918848f7b501da115d242d7e 2024-11-28T02:54:08,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/0ec39ad096364b53aaa457bb639cd9c3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/0ec39ad096364b53aaa457bb639cd9c3 2024-11-28T02:54:08,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d815be114d3d419a99af1327e88bee09 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/d815be114d3d419a99af1327e88bee09 2024-11-28T02:54:08,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/0bcf206322694e0ca93e9d9ea4f9be48 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/0bcf206322694e0ca93e9d9ea4f9be48 2024-11-28T02:54:08,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b3644782f8874e069d2f7f351a8ee9a1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/34b319e909364a80952f5e925097d0ee, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/551aa9501fd242d481c720a93eec00e7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/1e73d010bdff443ba35096d065ca0ba9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6be7b85ffc8840c988b1379f5940be45, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/25502e79940e46e3931e4155c8671dc4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/faa18605e45643f29983c731730bc035, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/cd7c186d06d941f989f6f4a68519b327, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/38992963d147417db213e2d2ff2f0359, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6bf6ba2bf67043e1bcc24a0e3c3420e7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/c0d47269dd0f4233b01d52019292d831, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/26aea37ed6b44fcda446d7028fdda2ca, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/ca5b2179b73f4925b4da78b6112a5485, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6c6e1706b8ae479fbb17c0f056bebda3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/bf29b1b5147943a0ad69fdf835136c95, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/e4b00749c6314a3bb0012157cf236762, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/21c3fb019a9d4505b47ad759ce4a25b4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/624416e9a2184636b5a17c3d2b92ba8b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/a30ddd2b100f456cb9e2e830bfaa2d30, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/7dba5e142d8d4815b2fe293a03c79859, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/f8fef764602c4193b8261bd63b99b85e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/d784d91dd4ec494b9b47e1f241ef62da, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/8b7d518ccbeb4c3c82c1e820e6090c5f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b201e2bf134e40dc835a22ba399246bd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/2b3242b13ab643bfbe0a369a979b505c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/71e470a1510a49aa82ede15ed2853f7a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b83b8c348e5b450f8d13ba380dfced9a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6830820587a24708b4399cd02e42abbc] to archive 2024-11-28T02:54:08,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:54:08,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b3644782f8874e069d2f7f351a8ee9a1 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b3644782f8874e069d2f7f351a8ee9a1 2024-11-28T02:54:08,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/34b319e909364a80952f5e925097d0ee to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/34b319e909364a80952f5e925097d0ee 2024-11-28T02:54:08,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/551aa9501fd242d481c720a93eec00e7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/551aa9501fd242d481c720a93eec00e7 2024-11-28T02:54:08,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/1e73d010bdff443ba35096d065ca0ba9 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/1e73d010bdff443ba35096d065ca0ba9 2024-11-28T02:54:08,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6be7b85ffc8840c988b1379f5940be45 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6be7b85ffc8840c988b1379f5940be45 2024-11-28T02:54:08,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/25502e79940e46e3931e4155c8671dc4 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/25502e79940e46e3931e4155c8671dc4 2024-11-28T02:54:08,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/faa18605e45643f29983c731730bc035 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/faa18605e45643f29983c731730bc035 2024-11-28T02:54:08,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/cd7c186d06d941f989f6f4a68519b327 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/cd7c186d06d941f989f6f4a68519b327 2024-11-28T02:54:08,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/38992963d147417db213e2d2ff2f0359 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/38992963d147417db213e2d2ff2f0359 2024-11-28T02:54:08,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6bf6ba2bf67043e1bcc24a0e3c3420e7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6bf6ba2bf67043e1bcc24a0e3c3420e7 2024-11-28T02:54:08,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/c0d47269dd0f4233b01d52019292d831 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/c0d47269dd0f4233b01d52019292d831 2024-11-28T02:54:08,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/26aea37ed6b44fcda446d7028fdda2ca to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/26aea37ed6b44fcda446d7028fdda2ca 2024-11-28T02:54:08,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/ca5b2179b73f4925b4da78b6112a5485 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/ca5b2179b73f4925b4da78b6112a5485 2024-11-28T02:54:08,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6c6e1706b8ae479fbb17c0f056bebda3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6c6e1706b8ae479fbb17c0f056bebda3 2024-11-28T02:54:08,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/bf29b1b5147943a0ad69fdf835136c95 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/bf29b1b5147943a0ad69fdf835136c95 2024-11-28T02:54:08,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/e4b00749c6314a3bb0012157cf236762 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/e4b00749c6314a3bb0012157cf236762 2024-11-28T02:54:08,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/21c3fb019a9d4505b47ad759ce4a25b4 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/21c3fb019a9d4505b47ad759ce4a25b4 2024-11-28T02:54:08,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/624416e9a2184636b5a17c3d2b92ba8b to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/624416e9a2184636b5a17c3d2b92ba8b 2024-11-28T02:54:08,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/a30ddd2b100f456cb9e2e830bfaa2d30 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/a30ddd2b100f456cb9e2e830bfaa2d30 2024-11-28T02:54:08,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/7dba5e142d8d4815b2fe293a03c79859 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/7dba5e142d8d4815b2fe293a03c79859 2024-11-28T02:54:08,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/f8fef764602c4193b8261bd63b99b85e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/f8fef764602c4193b8261bd63b99b85e 2024-11-28T02:54:08,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/d784d91dd4ec494b9b47e1f241ef62da to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/d784d91dd4ec494b9b47e1f241ef62da 2024-11-28T02:54:08,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/8b7d518ccbeb4c3c82c1e820e6090c5f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/8b7d518ccbeb4c3c82c1e820e6090c5f 2024-11-28T02:54:08,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b201e2bf134e40dc835a22ba399246bd to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b201e2bf134e40dc835a22ba399246bd 2024-11-28T02:54:08,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/2b3242b13ab643bfbe0a369a979b505c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/2b3242b13ab643bfbe0a369a979b505c 2024-11-28T02:54:08,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/71e470a1510a49aa82ede15ed2853f7a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/71e470a1510a49aa82ede15ed2853f7a 2024-11-28T02:54:08,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b83b8c348e5b450f8d13ba380dfced9a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b83b8c348e5b450f8d13ba380dfced9a 2024-11-28T02:54:08,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6830820587a24708b4399cd02e42abbc to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/6830820587a24708b4399cd02e42abbc 2024-11-28T02:54:08,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/52d6dd8eb9434dadbb35e854e93d91f7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/adb4907ea8e34c718969fae05f1ca2ea, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/52bb87d05c0141efa87297cee3a6f72e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/36b3866f271c4bbb9f3e0f28714a6542, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/ca8e6b9c9cc34f44834c262f58318e1f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/7f222cef905a4b96bb93b5daccfb24a6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4bb2f93c1a0d4ca99930dc736096152a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b41dd0f8adb84cf4a5376d158413cc37, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/19e3fc4ea94748f3a18efc436e9e75de, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/7e3910b920e44f6590eeb5993f4aa791, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/43658e1ceb674cce9d5cb3c463f0b14e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4111ce771aed48f296efdad4dbd656e2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/6471556c872a4349b2f7712481975e0d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0fba68832e614e8d8504f88b0c145ca1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/116e9a733bd64ef5a64d0c1294edb0d5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/455a40180b2e4766b9be42671bf90cd9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0e05c7129f28415a93f36707db2cca6c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/6d08d8c0e28d4987953b5920507c1e5a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4e8cfdc765ae45eda98b86edc232d169, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/39000a84580f409e99ad612df4db468a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/417bed8185a34006b97f912de1f6a02c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/067fde09fb87472cbc9e1dbb127590d7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0a432439bdc6411eb193da44e23b32b0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/a158045b656443bbbd3ab251ce83b417, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/216dc20916bb4b88912138f81ae8f9db, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/27582b36dd934cd69bc0b480a202d3af, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b2f8d7190e39435cbb7df10505aec5ea, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/84dd794e4b7446be963b1252494cfd8c] to archive 2024-11-28T02:54:08,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:54:08,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/52d6dd8eb9434dadbb35e854e93d91f7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/52d6dd8eb9434dadbb35e854e93d91f7 2024-11-28T02:54:08,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/adb4907ea8e34c718969fae05f1ca2ea to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/adb4907ea8e34c718969fae05f1ca2ea 2024-11-28T02:54:08,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/52bb87d05c0141efa87297cee3a6f72e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/52bb87d05c0141efa87297cee3a6f72e 2024-11-28T02:54:08,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/36b3866f271c4bbb9f3e0f28714a6542 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/36b3866f271c4bbb9f3e0f28714a6542 2024-11-28T02:54:08,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/ca8e6b9c9cc34f44834c262f58318e1f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/ca8e6b9c9cc34f44834c262f58318e1f 2024-11-28T02:54:08,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/7f222cef905a4b96bb93b5daccfb24a6 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/7f222cef905a4b96bb93b5daccfb24a6 2024-11-28T02:54:08,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4bb2f93c1a0d4ca99930dc736096152a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4bb2f93c1a0d4ca99930dc736096152a 2024-11-28T02:54:08,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b41dd0f8adb84cf4a5376d158413cc37 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b41dd0f8adb84cf4a5376d158413cc37 2024-11-28T02:54:08,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/19e3fc4ea94748f3a18efc436e9e75de to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/19e3fc4ea94748f3a18efc436e9e75de 2024-11-28T02:54:08,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/7e3910b920e44f6590eeb5993f4aa791 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/7e3910b920e44f6590eeb5993f4aa791 2024-11-28T02:54:08,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/43658e1ceb674cce9d5cb3c463f0b14e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/43658e1ceb674cce9d5cb3c463f0b14e 2024-11-28T02:54:08,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4111ce771aed48f296efdad4dbd656e2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4111ce771aed48f296efdad4dbd656e2 2024-11-28T02:54:08,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/6471556c872a4349b2f7712481975e0d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/6471556c872a4349b2f7712481975e0d 2024-11-28T02:54:08,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0fba68832e614e8d8504f88b0c145ca1 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0fba68832e614e8d8504f88b0c145ca1 2024-11-28T02:54:08,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/116e9a733bd64ef5a64d0c1294edb0d5 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/116e9a733bd64ef5a64d0c1294edb0d5 2024-11-28T02:54:08,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/455a40180b2e4766b9be42671bf90cd9 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/455a40180b2e4766b9be42671bf90cd9 2024-11-28T02:54:08,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0e05c7129f28415a93f36707db2cca6c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0e05c7129f28415a93f36707db2cca6c 2024-11-28T02:54:08,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/6d08d8c0e28d4987953b5920507c1e5a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/6d08d8c0e28d4987953b5920507c1e5a 2024-11-28T02:54:08,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4e8cfdc765ae45eda98b86edc232d169 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/4e8cfdc765ae45eda98b86edc232d169 2024-11-28T02:54:08,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/39000a84580f409e99ad612df4db468a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/39000a84580f409e99ad612df4db468a 2024-11-28T02:54:08,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/417bed8185a34006b97f912de1f6a02c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/417bed8185a34006b97f912de1f6a02c 2024-11-28T02:54:08,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/067fde09fb87472cbc9e1dbb127590d7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/067fde09fb87472cbc9e1dbb127590d7 2024-11-28T02:54:08,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0a432439bdc6411eb193da44e23b32b0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/0a432439bdc6411eb193da44e23b32b0 2024-11-28T02:54:08,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/a158045b656443bbbd3ab251ce83b417 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/a158045b656443bbbd3ab251ce83b417 2024-11-28T02:54:08,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/216dc20916bb4b88912138f81ae8f9db to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/216dc20916bb4b88912138f81ae8f9db 2024-11-28T02:54:08,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/27582b36dd934cd69bc0b480a202d3af to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/27582b36dd934cd69bc0b480a202d3af 2024-11-28T02:54:08,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b2f8d7190e39435cbb7df10505aec5ea to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b2f8d7190e39435cbb7df10505aec5ea 2024-11-28T02:54:08,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/84dd794e4b7446be963b1252494cfd8c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/84dd794e4b7446be963b1252494cfd8c 2024-11-28T02:54:08,121 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/recovered.edits/496.seqid, newMaxSeqId=496, maxSeqId=1 2024-11-28T02:54:08,121 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. 2024-11-28T02:54:08,121 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 1fec00bb9c12dd4e6236acd1fe2329f8: Waiting for close lock at 1732762446006Running coprocessor pre-close hooks at 1732762446006Disabling compacts and flushes for region at 1732762446006Disabling writes for close at 1732762447066 (+1060 ms)Obtaining lock to block concurrent updates at 1732762447066Preparing flush snapshotting stores in 1fec00bb9c12dd4e6236acd1fe2329f8 at 1732762447066Finished memstore snapshotting TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732762447067 (+1 ms)Flushing stores of TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8. at 1732762447067Flushing 1fec00bb9c12dd4e6236acd1fe2329f8/A: creating writer at 1732762447067Flushing 1fec00bb9c12dd4e6236acd1fe2329f8/A: appending metadata at 1732762447071 (+4 ms)Flushing 1fec00bb9c12dd4e6236acd1fe2329f8/A: closing flushed file at 1732762447071Flushing 1fec00bb9c12dd4e6236acd1fe2329f8/B: creating writer at 1732762447483 (+412 ms)Flushing 1fec00bb9c12dd4e6236acd1fe2329f8/B: appending metadata at 1732762447487 (+4 ms)Flushing 1fec00bb9c12dd4e6236acd1fe2329f8/B: closing flushed file at 1732762447487Flushing 1fec00bb9c12dd4e6236acd1fe2329f8/C: creating writer at 1732762447899 (+412 ms)Flushing 1fec00bb9c12dd4e6236acd1fe2329f8/C: appending metadata at 1732762447903 (+4 ms)Flushing 1fec00bb9c12dd4e6236acd1fe2329f8/C: closing flushed file at 1732762447904 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29695031: reopening flushed file at 1732762447923 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d8d487d: reopening flushed file at 1732762447930 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c5c2e51: reopening flushed file at 1732762447937 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1fec00bb9c12dd4e6236acd1fe2329f8 in 880ms, sequenceid=493, compaction requested=true at 1732762447946 (+9 ms)Writing region close event to WAL at 1732762448117 (+171 ms)Running coprocessor post-close hooks at 1732762448121 (+4 ms)Closed at 1732762448121 2024-11-28T02:54:08,124 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:54:08,124 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1fec00bb9c12dd4e6236acd1fe2329f8, regionState=CLOSED 2024-11-28T02:54:08,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:54:08,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-28T02:54:08,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 1fec00bb9c12dd4e6236acd1fe2329f8, server=978bd30859c3,41631,1732762419946 in 2.2780 sec 2024-11-28T02:54:08,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-28T02:54:08,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1fec00bb9c12dd4e6236acd1fe2329f8, UNASSIGN in 2.2850 sec 2024-11-28T02:54:08,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-28T02:54:08,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2920 sec 2024-11-28T02:54:08,133 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762448133"}]},"ts":"1732762448133"} 2024-11-28T02:54:08,135 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T02:54:08,135 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T02:54:08,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3070 sec 2024-11-28T02:54:09,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T02:54:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T02:54:10,006 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:10,009 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T02:54:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:10,014 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-28T02:54:10,016 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:10,020 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:54:10,023 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/recovered.edits] 2024-11-28T02:54:10,026 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/01520e839f9847f0b4cb9f323f6bd4bf to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/01520e839f9847f0b4cb9f323f6bd4bf 2024-11-28T02:54:10,028 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2803b62bdcf44e2194ea09acd9c3bd7f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/2803b62bdcf44e2194ea09acd9c3bd7f 2024-11-28T02:54:10,029 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/bdec06cde9c84f08a37ef6047c6243d3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/bdec06cde9c84f08a37ef6047c6243d3 2024-11-28T02:54:10,030 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/dc596c26590e40c396f3306c1254b4f0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/dc596c26590e40c396f3306c1254b4f0 2024-11-28T02:54:10,031 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/ebb8b154810a452ca7bb4f7677333962 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/A/ebb8b154810a452ca7bb4f7677333962 2024-11-28T02:54:10,033 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/15375fdce6d945f69673f39dcb534dfe to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/15375fdce6d945f69673f39dcb534dfe 2024-11-28T02:54:10,035 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/5a73998ee7654613aa3b268aedd33e88 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/5a73998ee7654613aa3b268aedd33e88 2024-11-28T02:54:10,036 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/7be314f9ff1e433fa16415df44338e41 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/7be314f9ff1e433fa16415df44338e41 2024-11-28T02:54:10,037 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b004e307ee3c4ef1bf4e144d1a233843 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/b004e307ee3c4ef1bf4e144d1a233843 2024-11-28T02:54:10,037 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/f1428bd9b9444bc683b9b9129fa5cf4c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/B/f1428bd9b9444bc683b9b9129fa5cf4c 2024-11-28T02:54:10,039 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/2bb3d494ba444db7995bbcf21bcf4461 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/2bb3d494ba444db7995bbcf21bcf4461 2024-11-28T02:54:10,040 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/322b7fae8ff34a71975b3e1ecd81d9ec to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/322b7fae8ff34a71975b3e1ecd81d9ec 2024-11-28T02:54:10,041 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/73b20fa27b064105b29def03c1f18d1d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/73b20fa27b064105b29def03c1f18d1d 2024-11-28T02:54:10,042 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b503cc3de19947f6964acf95f5e9bf3c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/b503cc3de19947f6964acf95f5e9bf3c 2024-11-28T02:54:10,044 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/d8f91f853abb446f80fb57007373ec5e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/C/d8f91f853abb446f80fb57007373ec5e 2024-11-28T02:54:10,047 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/recovered.edits/496.seqid to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8/recovered.edits/496.seqid 2024-11-28T02:54:10,047 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/1fec00bb9c12dd4e6236acd1fe2329f8 2024-11-28T02:54:10,048 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T02:54:10,053 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:10,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-28T02:54:10,066 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T02:54:10,100 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T02:54:10,101 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:10,101 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T02:54:10,102 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732762450101"}]},"ts":"9223372036854775807"} 2024-11-28T02:54:10,106 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T02:54:10,106 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1fec00bb9c12dd4e6236acd1fe2329f8, NAME => 'TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T02:54:10,106 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T02:54:10,106 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732762450106"}]},"ts":"9223372036854775807"} 2024-11-28T02:54:10,109 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T02:54:10,110 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:10,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 100 msec 2024-11-28T02:54:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-28T02:54:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T02:54:10,127 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:10,140 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1299054995_22 at /127.0.0.1:49994 [Waiting for operation #226] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1042822658_22 at /127.0.0.1:39250 [Waiting for operation #173] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/978bd30859c3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1299054995_22 at /127.0.0.1:39148 [Waiting for operation #214] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;978bd30859c3:41631-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=521 (was 352) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7684 (was 7457) - AvailableMemoryMB LEAK? - 2024-11-28T02:54:10,150 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=521, ProcessCount=11, AvailableMemoryMB=7683 2024-11-28T02:54:10,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T02:54:10,152 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:54:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:10,154 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T02:54:10,154 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:10,155 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-28T02:54:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-28T02:54:10,156 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T02:54:10,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741936_1112 (size=1061) 2024-11-28T02:54:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-28T02:54:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-28T02:54:10,573 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381 2024-11-28T02:54:10,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741937_1113 (size=53) 2024-11-28T02:54:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-28T02:54:10,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:54:10,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f5c4bfa8bb8330a5297e82be419743be, disabling compactions & flushes 2024-11-28T02:54:10,983 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:10,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:10,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. after waiting 0 ms 2024-11-28T02:54:10,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:10,983 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:10,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f5c4bfa8bb8330a5297e82be419743be: Waiting for close lock at 1732762450983Disabling compacts and flushes for region at 1732762450983Disabling writes for close at 1732762450983Writing region close event to WAL at 1732762450983Closed at 1732762450983 2024-11-28T02:54:10,984 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T02:54:10,985 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732762450984"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732762450984"}]},"ts":"1732762450984"} 2024-11-28T02:54:10,987 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T02:54:10,988 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T02:54:10,988 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762450988"}]},"ts":"1732762450988"} 2024-11-28T02:54:10,990 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T02:54:10,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, ASSIGN}] 2024-11-28T02:54:10,992 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, ASSIGN 2024-11-28T02:54:10,993 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, ASSIGN; state=OFFLINE, location=978bd30859c3,41631,1732762419946; forceNewPlan=false, retain=false 2024-11-28T02:54:11,144 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f5c4bfa8bb8330a5297e82be419743be, regionState=OPENING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:54:11,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, ASSIGN because future has completed 2024-11-28T02:54:11,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:54:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-28T02:54:11,303 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:11,303 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:54:11,304 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:11,304 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:54:11,304 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:11,304 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:11,306 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:11,307 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:54:11,308 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5c4bfa8bb8330a5297e82be419743be columnFamilyName A 2024-11-28T02:54:11,308 DEBUG [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:11,309 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.HStore(327): Store=f5c4bfa8bb8330a5297e82be419743be/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:54:11,309 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:11,311 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:54:11,311 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5c4bfa8bb8330a5297e82be419743be columnFamilyName B 2024-11-28T02:54:11,311 DEBUG [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:11,312 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.HStore(327): Store=f5c4bfa8bb8330a5297e82be419743be/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:54:11,312 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:11,314 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:54:11,314 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5c4bfa8bb8330a5297e82be419743be columnFamilyName C 2024-11-28T02:54:11,314 DEBUG [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:11,314 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.HStore(327): Store=f5c4bfa8bb8330a5297e82be419743be/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:54:11,315 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:11,315 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:11,316 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:11,316 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:11,317 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:11,317 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:11,318 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T02:54:11,319 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:11,322 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:54:11,322 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened f5c4bfa8bb8330a5297e82be419743be; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67879397, jitterRate=0.011481836438179016}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T02:54:11,323 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:11,323 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for f5c4bfa8bb8330a5297e82be419743be: Running coprocessor pre-open hook at 1732762451304Writing region info on filesystem at 1732762451304Initializing all the Stores at 1732762451305 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762451305Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762451305Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762451305Cleaning up temporary data from old regions at 1732762451317 (+12 ms)Running coprocessor post-open hooks at 1732762451323 (+6 ms)Region opened successfully at 1732762451323 2024-11-28T02:54:11,324 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., pid=32, masterSystemTime=1732762451298 2024-11-28T02:54:11,327 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:11,327 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:11,328 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f5c4bfa8bb8330a5297e82be419743be, regionState=OPEN, openSeqNum=2, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:54:11,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:54:11,333 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-28T02:54:11,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 in 185 msec 2024-11-28T02:54:11,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-28T02:54:11,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, ASSIGN in 342 msec 2024-11-28T02:54:11,336 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T02:54:11,337 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762451336"}]},"ts":"1732762451336"} 2024-11-28T02:54:11,339 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T02:54:11,339 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T02:54:11,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-11-28T02:54:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-28T02:54:12,307 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:12,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a00013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:12,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:12,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:12,311 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@212f8e21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:12,312 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e15a65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:12,313 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:12,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:12,317 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T02:54:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 978bd30859c3,32823,1732762419008 2024-11-28T02:54:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@66520796 2024-11-28T02:54:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T02:54:12,318 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T02:54:12,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T02:54:12,324 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:54:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:12,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741938_1114 (size=1097) 2024-11-28T02:54:12,744 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T02:54:12,744 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T02:54:12,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T02:54:12,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, REOPEN/MOVE}] 2024-11-28T02:54:12,760 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, REOPEN/MOVE 2024-11-28T02:54:12,761 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f5c4bfa8bb8330a5297e82be419743be, regionState=CLOSING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:54:12,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, REOPEN/MOVE because future has completed 2024-11-28T02:54:12,763 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T02:54:12,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:54:12,916 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:12,916 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T02:54:12,916 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing f5c4bfa8bb8330a5297e82be419743be, disabling compactions & flushes 2024-11-28T02:54:12,916 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:12,916 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:12,916 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. after waiting 0 ms 2024-11-28T02:54:12,916 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:12,921 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T02:54:12,922 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:12,922 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for f5c4bfa8bb8330a5297e82be419743be: Waiting for close lock at 1732762452916Running coprocessor pre-close hooks at 1732762452916Disabling compacts and flushes for region at 1732762452916Disabling writes for close at 1732762452916Writing region close event to WAL at 1732762452916Running coprocessor post-close hooks at 1732762452922 (+6 ms)Closed at 1732762452922 2024-11-28T02:54:12,922 WARN [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: f5c4bfa8bb8330a5297e82be419743be to self. 2024-11-28T02:54:12,925 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:12,925 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f5c4bfa8bb8330a5297e82be419743be, regionState=CLOSED 2024-11-28T02:54:12,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:54:12,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-28T02:54:12,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 in 165 msec 2024-11-28T02:54:12,931 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, REOPEN/MOVE; state=CLOSED, location=978bd30859c3,41631,1732762419946; forceNewPlan=false, retain=true 2024-11-28T02:54:13,082 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f5c4bfa8bb8330a5297e82be419743be, regionState=OPENING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, REOPEN/MOVE because future has completed 2024-11-28T02:54:13,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:54:13,240 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:13,240 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:54:13,241 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,241 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:54:13,241 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,241 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,245 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,246 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:54:13,250 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5c4bfa8bb8330a5297e82be419743be columnFamilyName A 2024-11-28T02:54:13,252 DEBUG [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:13,253 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.HStore(327): Store=f5c4bfa8bb8330a5297e82be419743be/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:54:13,253 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,254 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:54:13,254 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5c4bfa8bb8330a5297e82be419743be columnFamilyName B 2024-11-28T02:54:13,254 DEBUG [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:13,254 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.HStore(327): Store=f5c4bfa8bb8330a5297e82be419743be/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:54:13,254 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,255 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:54:13,255 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5c4bfa8bb8330a5297e82be419743be columnFamilyName C 2024-11-28T02:54:13,255 DEBUG [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:13,256 INFO [StoreOpener-f5c4bfa8bb8330a5297e82be419743be-1 {}] regionserver.HStore(327): Store=f5c4bfa8bb8330a5297e82be419743be/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:54:13,256 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:13,256 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,257 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,258 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,259 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,259 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,259 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T02:54:13,261 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,261 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened f5c4bfa8bb8330a5297e82be419743be; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68662362, jitterRate=0.023148924112319946}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T02:54:13,261 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,262 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for f5c4bfa8bb8330a5297e82be419743be: Running coprocessor pre-open hook at 1732762453241Writing region info on filesystem at 1732762453241Initializing all the Stores at 1732762453242 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762453242Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762453243 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762453243Cleaning up temporary data from old regions at 1732762453259 (+16 ms)Running coprocessor post-open hooks at 1732762453261 (+2 ms)Region opened successfully at 1732762453262 (+1 ms) 2024-11-28T02:54:13,263 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., pid=37, masterSystemTime=1732762453236 2024-11-28T02:54:13,265 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:13,265 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:13,265 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f5c4bfa8bb8330a5297e82be419743be, regionState=OPEN, openSeqNum=5, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:54:13,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-28T02:54:13,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 in 184 msec 2024-11-28T02:54:13,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-28T02:54:13,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, REOPEN/MOVE in 511 msec 2024-11-28T02:54:13,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-28T02:54:13,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 524 msec 2024-11-28T02:54:13,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 952 msec 2024-11-28T02:54:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-28T02:54:13,282 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:13,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad8b864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,283 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:13,283 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:13,284 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:13,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5902fb04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:13,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:13,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:13,286 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d9b2e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,288 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:13,288 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:13,289 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6455e7c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:13,291 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:13,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47b625fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:13,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:13,293 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:13,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@445cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:13,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:13,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:13,295 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435e9043, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:13,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:13,298 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b498ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:13,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:13,300 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:13,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4062c667, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:13,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:13,303 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d34b2ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:13,305 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ffc7e7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,307 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:13,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:13,307 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7de9ec7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:13,309 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@316f0081, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:13,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:13,311 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:13,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:13,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:13,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67a92d31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:13,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:13,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:13,313 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76fa478, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:13,314 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:13,315 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54894ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:13,316 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:13,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2490070f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:13,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:13,319 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:13,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4cc7ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:13,321 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3649d1c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:13,323 DEBUG [Thread-586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:13,323 DEBUG [Thread-592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:13,323 DEBUG [Thread-594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:13,323 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:13,323 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:13,324 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:13,325 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:13,325 DEBUG [Thread-599 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:13,326 DEBUG [Thread-599 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:13,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-28T02:54:13,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:13,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:13,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:13,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:13,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:13,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:13,328 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T02:54:13,329 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:13,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:13,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:54:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:54:13,332 DEBUG [Thread-588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:54:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:54:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:54:13,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:13,335 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:54:13,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:13,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:13,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:13,336 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:13,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:13,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:13,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:54:13,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:54:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:13,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:54:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:54:13,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:13,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:13,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:13,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:13,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:13,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:13,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128802a114b13294307a23549f54f6f09e4_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762453339/Put/seqid=0 2024-11-28T02:54:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T02:54:13,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741939_1115 (size=19474) 2024-11-28T02:54:13,459 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:13,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762453425, max timestamp=1732762453457Num uniques 859; 2024-11-28T02:54:13,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:54:13,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762513436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762453425, max timestamp=1732762453457Num uniques 859; 2024-11-28T02:54:13,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:54:13,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762453425, max timestamp=1732762453457Num uniques 859; 2024-11-28T02:54:13,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:54:13,472 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128802a114b13294307a23549f54f6f09e4_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128802a114b13294307a23549f54f6f09e4_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,475 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/45828bd44ca94de3be3e557d50cb8265, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:13,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T02:54:13,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:13,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:13,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:13,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:13,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:13,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/45828bd44ca94de3be3e557d50cb8265 is 175, key is test_row_0/A:col10/1732762453339/Put/seqid=0 2024-11-28T02:54:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:13,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741940_1116 (size=56733) 2024-11-28T02:54:13,523 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/45828bd44ca94de3be3e557d50cb8265 2024-11-28T02:54:13,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39476 deadline: 1732762513519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39434 deadline: 1732762513519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39452 deadline: 1732762513524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1732762513525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/1ab40fd58b42460a99c6cc53d974599c is 50, key is test_row_0/B:col10/1732762453339/Put/seqid=0 2024-11-28T02:54:13,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741941_1117 (size=12001) 2024-11-28T02:54:13,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/1ab40fd58b42460a99c6cc53d974599c 2024-11-28T02:54:13,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762513576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/2e22172147e34287b2a9b5e469644bf8 is 50, key is test_row_0/C:col10/1732762453339/Put/seqid=0 2024-11-28T02:54:13,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741942_1118 (size=12001) 2024-11-28T02:54:13,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/2e22172147e34287b2a9b5e469644bf8 2024-11-28T02:54:13,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T02:54:13,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:13,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:13,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:13,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:13,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:13,640 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T02:54:13,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39476 deadline: 1732762513636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39434 deadline: 1732762513636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T02:54:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39452 deadline: 1732762513636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1732762513636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/45828bd44ca94de3be3e557d50cb8265 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/45828bd44ca94de3be3e557d50cb8265 2024-11-28T02:54:13,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/45828bd44ca94de3be3e557d50cb8265, entries=300, sequenceid=15, filesize=55.4 K 2024-11-28T02:54:13,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/1ab40fd58b42460a99c6cc53d974599c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/1ab40fd58b42460a99c6cc53d974599c 2024-11-28T02:54:13,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/1ab40fd58b42460a99c6cc53d974599c, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T02:54:13,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/2e22172147e34287b2a9b5e469644bf8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/2e22172147e34287b2a9b5e469644bf8 2024-11-28T02:54:13,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/2e22172147e34287b2a9b5e469644bf8, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T02:54:13,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for f5c4bfa8bb8330a5297e82be419743be in 339ms, sequenceid=15, compaction requested=false 2024-11-28T02:54:13,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:13,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T02:54:13,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:13,792 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-28T02:54:13,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:13,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:13,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:13,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:13,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:13,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:13,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288ed74e4be70742738b7f556c7834aba0_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762453497/Put/seqid=0 2024-11-28T02:54:13,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741943_1119 (size=14594) 2024-11-28T02:54:13,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:13,823 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288ed74e4be70742738b7f556c7834aba0_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288ed74e4be70742738b7f556c7834aba0_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:13,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/60904d708f88473d878df59f370e4d17, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:13,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/60904d708f88473d878df59f370e4d17 is 175, key is test_row_0/A:col10/1732762453497/Put/seqid=0 2024-11-28T02:54:13,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741944_1120 (size=39549) 2024-11-28T02:54:13,844 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/60904d708f88473d878df59f370e4d17 2024-11-28T02:54:13,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762513847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39476 deadline: 1732762513856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39434 deadline: 1732762513857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39452 deadline: 1732762513857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1732762513860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/0c77e237756c45e7a383062d59798c11 is 50, key is test_row_0/B:col10/1732762453497/Put/seqid=0 2024-11-28T02:54:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:13,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741945_1121 (size=12001) 2024-11-28T02:54:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T02:54:13,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762513957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:13,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:14,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39452 deadline: 1732762514166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:14,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762514166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:14,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:14,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:14,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39434 deadline: 1732762514177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:14,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1732762514177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:14,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39476 deadline: 1732762514177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:14,281 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/0c77e237756c45e7a383062d59798c11 2024-11-28T02:54:14,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/4aeec579b9cf456685a0cb1faea34be7 is 50, key is test_row_0/C:col10/1732762453497/Put/seqid=0 2024-11-28T02:54:14,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741946_1122 (size=12001) 2024-11-28T02:54:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T02:54:14,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762514486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:14,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:14,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39452 deadline: 1732762514686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1732762514686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39434 deadline: 1732762514687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39476 deadline: 1732762514687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:14,729 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/4aeec579b9cf456685a0cb1faea34be7 2024-11-28T02:54:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/60904d708f88473d878df59f370e4d17 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/60904d708f88473d878df59f370e4d17 2024-11-28T02:54:14,745 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/60904d708f88473d878df59f370e4d17, entries=200, sequenceid=51, filesize=38.6 K 2024-11-28T02:54:14,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/0c77e237756c45e7a383062d59798c11 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/0c77e237756c45e7a383062d59798c11 2024-11-28T02:54:14,754 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/0c77e237756c45e7a383062d59798c11, entries=150, sequenceid=51, filesize=11.7 K 2024-11-28T02:54:14,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/4aeec579b9cf456685a0cb1faea34be7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4aeec579b9cf456685a0cb1faea34be7 2024-11-28T02:54:14,763 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4aeec579b9cf456685a0cb1faea34be7, entries=150, sequenceid=51, filesize=11.7 K 2024-11-28T02:54:14,764 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for f5c4bfa8bb8330a5297e82be419743be in 972ms, sequenceid=51, compaction requested=false 2024-11-28T02:54:14,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:14,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:14,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-28T02:54:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-28T02:54:14,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-28T02:54:14,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4370 sec 2024-11-28T02:54:14,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.4460 sec 2024-11-28T02:54:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:14,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T02:54:14,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:14,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:14,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:14,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:14,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:14,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:15,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288ea784e996864d7fa76d44abdb3f9e3c_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762453838/Put/seqid=0 2024-11-28T02:54:15,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741947_1123 (size=12154) 2024-11-28T02:54:15,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,057 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288ea784e996864d7fa76d44abdb3f9e3c_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288ea784e996864d7fa76d44abdb3f9e3c_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:15,059 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/b4c7af5dad7940f3836c7294e626f6d4, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:15,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/b4c7af5dad7940f3836c7294e626f6d4 is 175, key is test_row_0/A:col10/1732762453838/Put/seqid=0 2024-11-28T02:54:15,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741948_1124 (size=30955) 2024-11-28T02:54:15,076 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/b4c7af5dad7940f3836c7294e626f6d4 2024-11-28T02:54:15,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/e355bc209020404d94f0792df72a6773 is 50, key is test_row_0/B:col10/1732762453838/Put/seqid=0 2024-11-28T02:54:15,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762455048, max timestamp=1732762455102Num uniques 859; 2024-11-28T02:54:15,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762455048, max timestamp=1732762455102Num uniques 859; 2024-11-28T02:54:15,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:54:15,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:54:15,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762455048, max timestamp=1732762455102Num uniques 859; 2024-11-28T02:54:15,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:15,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741949_1125 (size=12001) 2024-11-28T02:54:15,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/e355bc209020404d94f0792df72a6773 2024-11-28T02:54:15,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/2c2b3d29610c43dc951a23cfebaf5a9a is 50, key is test_row_0/C:col10/1732762453838/Put/seqid=0 2024-11-28T02:54:15,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741950_1126 (size=12001) 2024-11-28T02:54:15,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/2c2b3d29610c43dc951a23cfebaf5a9a 2024-11-28T02:54:15,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/b4c7af5dad7940f3836c7294e626f6d4 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/b4c7af5dad7940f3836c7294e626f6d4 2024-11-28T02:54:15,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/b4c7af5dad7940f3836c7294e626f6d4, entries=150, sequenceid=61, filesize=30.2 K 2024-11-28T02:54:15,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/e355bc209020404d94f0792df72a6773 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/e355bc209020404d94f0792df72a6773 2024-11-28T02:54:15,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/e355bc209020404d94f0792df72a6773, entries=150, sequenceid=61, filesize=11.7 K 2024-11-28T02:54:15,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/2c2b3d29610c43dc951a23cfebaf5a9a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/2c2b3d29610c43dc951a23cfebaf5a9a 2024-11-28T02:54:15,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/2c2b3d29610c43dc951a23cfebaf5a9a, entries=150, sequenceid=61, filesize=11.7 K 2024-11-28T02:54:15,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=125.23 KB/128240 for f5c4bfa8bb8330a5297e82be419743be in 192ms, sequenceid=61, compaction requested=true 2024-11-28T02:54:15,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:15,191 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:15,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:15,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:15,191 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:15,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:15,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:15,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:15,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:15,192 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:15,192 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:15,192 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/A is initiating minor compaction (all files) 2024-11-28T02:54:15,192 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/B is initiating minor compaction (all files) 2024-11-28T02:54:15,193 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/B in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:15,193 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/A in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:15,193 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/45828bd44ca94de3be3e557d50cb8265, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/60904d708f88473d878df59f370e4d17, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/b4c7af5dad7940f3836c7294e626f6d4] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=124.3 K 2024-11-28T02:54:15,193 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/1ab40fd58b42460a99c6cc53d974599c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/0c77e237756c45e7a383062d59798c11, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/e355bc209020404d94f0792df72a6773] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=35.2 K 2024-11-28T02:54:15,193 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:15,193 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/45828bd44ca94de3be3e557d50cb8265, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/60904d708f88473d878df59f370e4d17, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/b4c7af5dad7940f3836c7294e626f6d4] 2024-11-28T02:54:15,193 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ab40fd58b42460a99c6cc53d974599c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732762453339 2024-11-28T02:54:15,194 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45828bd44ca94de3be3e557d50cb8265, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732762453333 2024-11-28T02:54:15,194 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c77e237756c45e7a383062d59798c11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732762453497 2024-11-28T02:54:15,194 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60904d708f88473d878df59f370e4d17, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732762453497 2024-11-28T02:54:15,194 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting e355bc209020404d94f0792df72a6773, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732762453838 2024-11-28T02:54:15,194 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4c7af5dad7940f3836c7294e626f6d4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732762453838 2024-11-28T02:54:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:15,201 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:15,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-11-28T02:54:15,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:15,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:15,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:15,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:15,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:15,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:15,210 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#B#compaction#108 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:15,211 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/6bed607304184e8386213ed2814fcdfc is 50, key is test_row_0/B:col10/1732762453838/Put/seqid=0 2024-11-28T02:54:15,219 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411284badcecaf7424018811d1090650e9c54_f5c4bfa8bb8330a5297e82be419743be store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:15,226 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411284badcecaf7424018811d1090650e9c54_f5c4bfa8bb8330a5297e82be419743be, store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:15,227 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284badcecaf7424018811d1090650e9c54_f5c4bfa8bb8330a5297e82be419743be because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:15,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283daf31618ca04af089f8495a2b5902b9_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762455177/Put/seqid=0 2024-11-28T02:54:15,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741951_1127 (size=12104) 2024-11-28T02:54:15,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741952_1128 (size=4469) 2024-11-28T02:54:15,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741953_1129 (size=17034) 2024-11-28T02:54:15,273 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762515277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:15,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:15,280 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283daf31618ca04af089f8495a2b5902b9_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283daf31618ca04af089f8495a2b5902b9_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:15,282 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e0bd4a20d91648a3a74d60040b14b0f4, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:15,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e0bd4a20d91648a3a74d60040b14b0f4 is 175, key is test_row_0/A:col10/1732762455177/Put/seqid=0 2024-11-28T02:54:15,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741954_1130 (size=48139) 2024-11-28T02:54:15,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762515386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T02:54:15,476 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:15,477 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-28T02:54:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T02:54:15,479 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:15,479 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:15,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T02:54:15,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762515596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:15,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:15,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T02:54:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:15,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:15,659 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/6bed607304184e8386213ed2814fcdfc as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/6bed607304184e8386213ed2814fcdfc 2024-11-28T02:54:15,660 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#A#compaction#109 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:15,661 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/0ae1628250f8465c97008d3b50dae6a2 is 175, key is test_row_0/A:col10/1732762453838/Put/seqid=0 2024-11-28T02:54:15,670 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/B of f5c4bfa8bb8330a5297e82be419743be into 6bed607304184e8386213ed2814fcdfc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:15,670 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:15,670 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/B, priority=13, startTime=1732762455191; duration=0sec 2024-11-28T02:54:15,671 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:15,671 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:B 2024-11-28T02:54:15,671 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:15,674 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:15,674 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/C is initiating minor compaction (all files) 2024-11-28T02:54:15,674 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/C in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:15,674 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/2e22172147e34287b2a9b5e469644bf8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4aeec579b9cf456685a0cb1faea34be7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/2c2b3d29610c43dc951a23cfebaf5a9a] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=35.2 K 2024-11-28T02:54:15,675 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e22172147e34287b2a9b5e469644bf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732762453339 2024-11-28T02:54:15,675 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aeec579b9cf456685a0cb1faea34be7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732762453497 2024-11-28T02:54:15,675 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c2b3d29610c43dc951a23cfebaf5a9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732762453838 2024-11-28T02:54:15,683 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#C#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:15,684 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/44102efe408a4f4381cc5e428dbfb15c is 50, key is test_row_0/C:col10/1732762453838/Put/seqid=0 2024-11-28T02:54:15,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741955_1131 (size=31058) 2024-11-28T02:54:15,699 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/0ae1628250f8465c97008d3b50dae6a2 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/0ae1628250f8465c97008d3b50dae6a2 2024-11-28T02:54:15,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39434 deadline: 1732762515706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:15,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39476 deadline: 1732762515706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:15,710 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:15,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39452 deadline: 1732762515707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:15,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1732762515707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:15,712 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/A of f5c4bfa8bb8330a5297e82be419743be into 0ae1628250f8465c97008d3b50dae6a2(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:15,712 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:15,712 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e0bd4a20d91648a3a74d60040b14b0f4 2024-11-28T02:54:15,712 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/A, priority=13, startTime=1732762455191; duration=0sec 2024-11-28T02:54:15,711 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:15,711 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_1' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:15,712 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:15,713 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:15,715 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:A 2024-11-28T02:54:15,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741956_1132 (size=12104) 2024-11-28T02:54:15,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/f4df4bf26c2047899ffebd8ed143019f is 50, key is test_row_0/B:col10/1732762455177/Put/seqid=0 2024-11-28T02:54:15,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741957_1133 (size=12001) 2024-11-28T02:54:15,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/f4df4bf26c2047899ffebd8ed143019f 2024-11-28T02:54:15,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/4b4427953a3145d38af7ba40b54ee3b0 is 50, key is test_row_0/C:col10/1732762455177/Put/seqid=0 2024-11-28T02:54:15,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741958_1134 (size=12001) 2024-11-28T02:54:15,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/4b4427953a3145d38af7ba40b54ee3b0 2024-11-28T02:54:15,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e0bd4a20d91648a3a74d60040b14b0f4 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e0bd4a20d91648a3a74d60040b14b0f4 2024-11-28T02:54:15,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T02:54:15,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:15,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:15,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:15,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:15,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:15,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e0bd4a20d91648a3a74d60040b14b0f4, entries=250, sequenceid=97, filesize=47.0 K 2024-11-28T02:54:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T02:54:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/f4df4bf26c2047899ffebd8ed143019f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/f4df4bf26c2047899ffebd8ed143019f 2024-11-28T02:54:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/f4df4bf26c2047899ffebd8ed143019f, entries=150, sequenceid=97, filesize=11.7 K 2024-11-28T02:54:15,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/4b4427953a3145d38af7ba40b54ee3b0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4b4427953a3145d38af7ba40b54ee3b0 2024-11-28T02:54:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4b4427953a3145d38af7ba40b54ee3b0, entries=150, sequenceid=97, filesize=11.7 K 2024-11-28T02:54:15,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for f5c4bfa8bb8330a5297e82be419743be in 620ms, sequenceid=97, compaction requested=false 2024-11-28T02:54:15,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:15,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:54:15,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:15,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:15,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:15,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:15,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:15,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T02:54:15,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:15,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:15,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:15,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:15,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fa4ad5d662b44d5cab4eaaa2249f028a_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762455938/Put/seqid=0 2024-11-28T02:54:15,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741959_1135 (size=12154) 2024-11-28T02:54:15,980 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:15,984 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fa4ad5d662b44d5cab4eaaa2249f028a_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fa4ad5d662b44d5cab4eaaa2249f028a_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:15,985 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/8f36c9ad16394f41b261f1ea2b9ae53e, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:15,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/8f36c9ad16394f41b261f1ea2b9ae53e is 175, key is test_row_0/A:col10/1732762455938/Put/seqid=0 2024-11-28T02:54:16,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741960_1136 (size=30955) 2024-11-28T02:54:16,014 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/8f36c9ad16394f41b261f1ea2b9ae53e 2024-11-28T02:54:16,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/da9d5788c25f4b6599c0fab6e0330c81 is 50, key is test_row_0/B:col10/1732762455938/Put/seqid=0 2024-11-28T02:54:16,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741961_1137 (size=12001) 2024-11-28T02:54:16,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/da9d5788c25f4b6599c0fab6e0330c81 2024-11-28T02:54:16,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T02:54:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T02:54:16,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762456043, max timestamp=1732762456092Num uniques 859; 2024-11-28T02:54:16,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762456043, max timestamp=1732762456092Num uniques 859; 2024-11-28T02:54:16,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762456043, max timestamp=1732762456092Num uniques 859; 2024-11-28T02:54:16,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:16,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:16,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:16,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/6625f1bf8ac74ec78326212ed68c58cb is 50, key is test_row_0/C:col10/1732762455938/Put/seqid=0 2024-11-28T02:54:16,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741962_1138 (size=12001) 2024-11-28T02:54:16,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/6625f1bf8ac74ec78326212ed68c58cb 2024-11-28T02:54:16,143 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/44102efe408a4f4381cc5e428dbfb15c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/44102efe408a4f4381cc5e428dbfb15c 2024-11-28T02:54:16,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/8f36c9ad16394f41b261f1ea2b9ae53e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/8f36c9ad16394f41b261f1ea2b9ae53e 2024-11-28T02:54:16,152 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/C of f5c4bfa8bb8330a5297e82be419743be into 44102efe408a4f4381cc5e428dbfb15c(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:16,152 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:16,152 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/C, priority=13, startTime=1732762455192; duration=0sec 2024-11-28T02:54:16,152 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:16,152 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:C 2024-11-28T02:54:16,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/8f36c9ad16394f41b261f1ea2b9ae53e, entries=150, sequenceid=110, filesize=30.2 K 2024-11-28T02:54:16,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/da9d5788c25f4b6599c0fab6e0330c81 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/da9d5788c25f4b6599c0fab6e0330c81 2024-11-28T02:54:16,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/da9d5788c25f4b6599c0fab6e0330c81, entries=150, sequenceid=110, filesize=11.7 K 2024-11-28T02:54:16,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/6625f1bf8ac74ec78326212ed68c58cb as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/6625f1bf8ac74ec78326212ed68c58cb 2024-11-28T02:54:16,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/6625f1bf8ac74ec78326212ed68c58cb, entries=150, sequenceid=110, filesize=11.7 K 2024-11-28T02:54:16,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=67.09 KB/68700 for f5c4bfa8bb8330a5297e82be419743be in 237ms, sequenceid=110, compaction requested=true 2024-11-28T02:54:16,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:16,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:16,178 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:16,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:16,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:16,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:16,178 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:16,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:16,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:16,179 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:16,179 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/A is initiating minor compaction (all files) 2024-11-28T02:54:16,179 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/A in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,179 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/0ae1628250f8465c97008d3b50dae6a2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e0bd4a20d91648a3a74d60040b14b0f4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/8f36c9ad16394f41b261f1ea2b9ae53e] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=107.6 K 2024-11-28T02:54:16,180 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,180 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/0ae1628250f8465c97008d3b50dae6a2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e0bd4a20d91648a3a74d60040b14b0f4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/8f36c9ad16394f41b261f1ea2b9ae53e] 2024-11-28T02:54:16,180 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:16,180 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/B is initiating minor compaction (all files) 2024-11-28T02:54:16,180 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/B in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:16,180 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/6bed607304184e8386213ed2814fcdfc, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/f4df4bf26c2047899ffebd8ed143019f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/da9d5788c25f4b6599c0fab6e0330c81] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=35.3 K 2024-11-28T02:54:16,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=73.80 KB heapSize=205.69 KB 2024-11-28T02:54:16,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:16,181 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ae1628250f8465c97008d3b50dae6a2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732762453838 2024-11-28T02:54:16,181 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bed607304184e8386213ed2814fcdfc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732762453838 2024-11-28T02:54:16,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:16,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:16,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:16,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:16,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:16,181 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting f4df4bf26c2047899ffebd8ed143019f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732762455177 2024-11-28T02:54:16,181 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0bd4a20d91648a3a74d60040b14b0f4, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732762455161 2024-11-28T02:54:16,181 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f36c9ad16394f41b261f1ea2b9ae53e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762455242 2024-11-28T02:54:16,181 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting da9d5788c25f4b6599c0fab6e0330c81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762455242 2024-11-28T02:54:16,207 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:16,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a7a7126b62f2433dbfd379649a8c3394_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762456148/Put/seqid=0 2024-11-28T02:54:16,213 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#B#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:16,214 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/337a56282f4e43e9bfb2ceb77db13da9 is 50, key is test_row_0/B:col10/1732762455938/Put/seqid=0 2024-11-28T02:54:16,215 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128ae1d044d5626417ebd9bdb28de508e40_f5c4bfa8bb8330a5297e82be419743be store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:16,218 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128ae1d044d5626417ebd9bdb28de508e40_f5c4bfa8bb8330a5297e82be419743be, store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:16,218 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ae1d044d5626417ebd9bdb28de508e40_f5c4bfa8bb8330a5297e82be419743be because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:16,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T02:54:16,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:16,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741964_1140 (size=14794) 2024-11-28T02:54:16,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741963_1139 (size=12207) 2024-11-28T02:54:16,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741965_1141 (size=4469) 2024-11-28T02:54:16,265 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#A#compaction#118 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:16,266 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/9c9928e2359d46f5902f62d56b2422b5 is 175, key is test_row_0/A:col10/1732762455938/Put/seqid=0 2024-11-28T02:54:16,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741966_1142 (size=31161) 2024-11-28T02:54:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762516305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:16,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T02:54:16,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:16,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762516416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:16,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:16,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:16,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:16,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T02:54:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T02:54:16,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762516626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:16,661 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:16,670 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a7a7126b62f2433dbfd379649a8c3394_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a7a7126b62f2433dbfd379649a8c3394_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:16,672 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e8be2cd7531449e682ad86319ebbc8a3, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:16,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e8be2cd7531449e682ad86319ebbc8a3 is 175, key is test_row_0/A:col10/1732762456148/Put/seqid=0 2024-11-28T02:54:16,673 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/337a56282f4e43e9bfb2ceb77db13da9 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/337a56282f4e43e9bfb2ceb77db13da9 2024-11-28T02:54:16,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741967_1143 (size=39749) 2024-11-28T02:54:16,681 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e8be2cd7531449e682ad86319ebbc8a3 2024-11-28T02:54:16,683 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/B of f5c4bfa8bb8330a5297e82be419743be into 337a56282f4e43e9bfb2ceb77db13da9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:16,683 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:16,683 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/B, priority=13, startTime=1732762456178; duration=0sec 2024-11-28T02:54:16,684 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:16,684 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:B 2024-11-28T02:54:16,684 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:16,685 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:16,686 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/C is initiating minor compaction (all files) 2024-11-28T02:54:16,686 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/C in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,686 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/9c9928e2359d46f5902f62d56b2422b5 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/9c9928e2359d46f5902f62d56b2422b5 2024-11-28T02:54:16,686 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/44102efe408a4f4381cc5e428dbfb15c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4b4427953a3145d38af7ba40b54ee3b0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/6625f1bf8ac74ec78326212ed68c58cb] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=35.3 K 2024-11-28T02:54:16,687 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 44102efe408a4f4381cc5e428dbfb15c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732762453838 2024-11-28T02:54:16,687 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b4427953a3145d38af7ba40b54ee3b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732762455177 2024-11-28T02:54:16,688 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6625f1bf8ac74ec78326212ed68c58cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762455242 2024-11-28T02:54:16,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/2b6382263589406e9bcdd26d4a76c1af is 50, key is test_row_0/B:col10/1732762456148/Put/seqid=0 2024-11-28T02:54:16,693 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/A of f5c4bfa8bb8330a5297e82be419743be into 9c9928e2359d46f5902f62d56b2422b5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:16,693 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:16,693 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/A, priority=13, startTime=1732762456178; duration=0sec 2024-11-28T02:54:16,694 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:16,694 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:A 2024-11-28T02:54:16,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741968_1144 (size=12151) 2024-11-28T02:54:16,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/2b6382263589406e9bcdd26d4a76c1af 2024-11-28T02:54:16,709 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#C#compaction#121 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:16,710 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/145709b5da844988ad28b8d9a5e2ba72 is 50, key is test_row_0/C:col10/1732762455938/Put/seqid=0 2024-11-28T02:54:16,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/56ff90710f504e93a77636bbd64f0513 is 50, key is test_row_0/C:col10/1732762456148/Put/seqid=0 2024-11-28T02:54:16,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T02:54:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741970_1146 (size=12151) 2024-11-28T02:54:16,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741969_1145 (size=12207) 2024-11-28T02:54:16,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T02:54:16,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:16,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:16,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:16,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762516936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:16,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:17,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T02:54:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:17,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:17,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:17,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/56ff90710f504e93a77636bbd64f0513 2024-11-28T02:54:17,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e8be2cd7531449e682ad86319ebbc8a3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e8be2cd7531449e682ad86319ebbc8a3 2024-11-28T02:54:17,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e8be2cd7531449e682ad86319ebbc8a3, entries=200, sequenceid=139, filesize=38.8 K 2024-11-28T02:54:17,146 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/145709b5da844988ad28b8d9a5e2ba72 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/145709b5da844988ad28b8d9a5e2ba72 2024-11-28T02:54:17,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/2b6382263589406e9bcdd26d4a76c1af as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/2b6382263589406e9bcdd26d4a76c1af 2024-11-28T02:54:17,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/2b6382263589406e9bcdd26d4a76c1af, entries=150, sequenceid=139, filesize=11.9 K 2024-11-28T02:54:17,152 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/C of f5c4bfa8bb8330a5297e82be419743be into 145709b5da844988ad28b8d9a5e2ba72(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:17,152 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:17,152 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/C, priority=13, startTime=1732762456178; duration=0sec 2024-11-28T02:54:17,152 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:17,152 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:C 2024-11-28T02:54:17,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/56ff90710f504e93a77636bbd64f0513 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/56ff90710f504e93a77636bbd64f0513 2024-11-28T02:54:17,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/56ff90710f504e93a77636bbd64f0513, entries=150, sequenceid=139, filesize=11.9 K 2024-11-28T02:54:17,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~205.64 KB/210576, currentSize=107.34 KB/109920 for f5c4bfa8bb8330a5297e82be419743be in 978ms, sequenceid=139, compaction requested=false 2024-11-28T02:54:17,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:17,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T02:54:17,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:17,179 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T02:54:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:17,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ae4d213238c645819d28ecaf7e251289_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762456253/Put/seqid=0 2024-11-28T02:54:17,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741971_1147 (size=12304) 2024-11-28T02:54:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:17,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:17,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762517488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:17,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:17,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762517595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:17,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:17,603 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ae4d213238c645819d28ecaf7e251289_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ae4d213238c645819d28ecaf7e251289_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:17,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e106156a114d42bc93f216a86a8e4171, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:17,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e106156a114d42bc93f216a86a8e4171 is 175, key is test_row_0/A:col10/1732762456253/Put/seqid=0 2024-11-28T02:54:17,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741972_1148 (size=31105) 2024-11-28T02:54:17,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T02:54:17,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39476 deadline: 1732762517726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:17,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39452 deadline: 1732762517726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:17,729 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:17,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:17,730 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_1' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:17,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:17,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39434 deadline: 1732762517736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:17,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1732762517736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:17,737 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:17,738 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:17,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762517806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:18,013 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e106156a114d42bc93f216a86a8e4171 2024-11-28T02:54:18,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/fe083d19e0154189a2e9a1e542ea4581 is 50, key is test_row_0/B:col10/1732762456253/Put/seqid=0 2024-11-28T02:54:18,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741973_1149 (size=12151) 2024-11-28T02:54:18,082 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/fe083d19e0154189a2e9a1e542ea4581 2024-11-28T02:54:18,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/50bbe96ce9e843588b398fe3c926717f is 50, key is test_row_0/C:col10/1732762456253/Put/seqid=0 2024-11-28T02:54:18,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741974_1150 (size=12151) 2024-11-28T02:54:18,104 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/50bbe96ce9e843588b398fe3c926717f 2024-11-28T02:54:18,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e106156a114d42bc93f216a86a8e4171 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e106156a114d42bc93f216a86a8e4171 2024-11-28T02:54:18,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:18,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762518116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:18,121 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e106156a114d42bc93f216a86a8e4171, entries=150, sequenceid=161, filesize=30.4 K 2024-11-28T02:54:18,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/fe083d19e0154189a2e9a1e542ea4581 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/fe083d19e0154189a2e9a1e542ea4581 2024-11-28T02:54:18,133 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/fe083d19e0154189a2e9a1e542ea4581, entries=150, sequenceid=161, filesize=11.9 K 2024-11-28T02:54:18,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/50bbe96ce9e843588b398fe3c926717f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/50bbe96ce9e843588b398fe3c926717f 2024-11-28T02:54:18,143 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/50bbe96ce9e843588b398fe3c926717f, entries=150, sequenceid=161, filesize=11.9 K 2024-11-28T02:54:18,145 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f5c4bfa8bb8330a5297e82be419743be in 967ms, sequenceid=161, compaction requested=true 2024-11-28T02:54:18,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:18,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:18,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-28T02:54:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-28T02:54:18,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-28T02:54:18,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6690 sec 2024-11-28T02:54:18,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.6750 sec 2024-11-28T02:54:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:18,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T02:54:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:18,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:18,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112893ffe98aa22a4a1c8b872e88ad6491f1_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762457481/Put/seqid=0 2024-11-28T02:54:18,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741975_1151 (size=14794) 2024-11-28T02:54:18,669 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:18,675 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112893ffe98aa22a4a1c8b872e88ad6491f1_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112893ffe98aa22a4a1c8b872e88ad6491f1_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:18,676 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/3255f0947e9240349b3ada0097377172, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:18,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/3255f0947e9240349b3ada0097377172 is 175, key is test_row_0/A:col10/1732762457481/Put/seqid=0 2024-11-28T02:54:18,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741976_1152 (size=39749) 2024-11-28T02:54:18,692 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/3255f0947e9240349b3ada0097377172 2024-11-28T02:54:18,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/3ce6433a30f94b3d991d2ce1c77beb72 is 50, key is test_row_0/B:col10/1732762457481/Put/seqid=0 2024-11-28T02:54:18,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762518717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:18,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:18,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:18,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:18,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741977_1153 (size=12151) 2024-11-28T02:54:18,743 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T02:54:18,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762518826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:18,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:18,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:18,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:19,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762519035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:19,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:19,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:19,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:19,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/3ce6433a30f94b3d991d2ce1c77beb72 2024-11-28T02:54:19,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/36563d1d1aa4407b98165ba152877940 is 50, key is test_row_0/C:col10/1732762457481/Put/seqid=0 2024-11-28T02:54:19,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741978_1154 (size=12151) 2024-11-28T02:54:19,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/36563d1d1aa4407b98165ba152877940 2024-11-28T02:54:19,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/3255f0947e9240349b3ada0097377172 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3255f0947e9240349b3ada0097377172 2024-11-28T02:54:19,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3255f0947e9240349b3ada0097377172, entries=200, sequenceid=175, filesize=38.8 K 2024-11-28T02:54:19,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/3ce6433a30f94b3d991d2ce1c77beb72 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/3ce6433a30f94b3d991d2ce1c77beb72 2024-11-28T02:54:19,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/3ce6433a30f94b3d991d2ce1c77beb72, entries=150, sequenceid=175, filesize=11.9 K 2024-11-28T02:54:19,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/36563d1d1aa4407b98165ba152877940 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/36563d1d1aa4407b98165ba152877940 2024-11-28T02:54:19,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/36563d1d1aa4407b98165ba152877940, entries=150, sequenceid=175, filesize=11.9 K 2024-11-28T02:54:19,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f5c4bfa8bb8330a5297e82be419743be in 544ms, sequenceid=175, compaction requested=true 2024-11-28T02:54:19,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:19,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:19,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:19,173 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:54:19,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:19,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:19,173 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:54:19,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:19,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:19,174 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141764 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:54:19,174 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:54:19,174 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/B is initiating minor compaction (all files) 2024-11-28T02:54:19,174 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/A is initiating minor compaction (all files) 2024-11-28T02:54:19,174 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/A in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:19,174 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/B in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:19,174 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/337a56282f4e43e9bfb2ceb77db13da9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/2b6382263589406e9bcdd26d4a76c1af, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/fe083d19e0154189a2e9a1e542ea4581, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/3ce6433a30f94b3d991d2ce1c77beb72] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=47.5 K 2024-11-28T02:54:19,174 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/9c9928e2359d46f5902f62d56b2422b5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e8be2cd7531449e682ad86319ebbc8a3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e106156a114d42bc93f216a86a8e4171, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3255f0947e9240349b3ada0097377172] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=138.4 K 2024-11-28T02:54:19,174 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:19,174 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/9c9928e2359d46f5902f62d56b2422b5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e8be2cd7531449e682ad86319ebbc8a3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e106156a114d42bc93f216a86a8e4171, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3255f0947e9240349b3ada0097377172] 2024-11-28T02:54:19,177 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 337a56282f4e43e9bfb2ceb77db13da9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762455242 2024-11-28T02:54:19,178 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b6382263589406e9bcdd26d4a76c1af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732762456148 2024-11-28T02:54:19,178 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c9928e2359d46f5902f62d56b2422b5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762455242 2024-11-28T02:54:19,180 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting fe083d19e0154189a2e9a1e542ea4581, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732762456253 2024-11-28T02:54:19,180 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8be2cd7531449e682ad86319ebbc8a3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732762456116 2024-11-28T02:54:19,181 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e106156a114d42bc93f216a86a8e4171, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732762456253 2024-11-28T02:54:19,181 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ce6433a30f94b3d991d2ce1c77beb72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732762457471 2024-11-28T02:54:19,181 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3255f0947e9240349b3ada0097377172, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732762457471 2024-11-28T02:54:19,205 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:19,206 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/9ca89d38855f4bba89dc8f4e26491c2c is 50, key is test_row_0/B:col10/1732762457481/Put/seqid=0 2024-11-28T02:54:19,208 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:19,212 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128bfed2856c31342439159341cb88ec430_f5c4bfa8bb8330a5297e82be419743be store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:19,215 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128bfed2856c31342439159341cb88ec430_f5c4bfa8bb8330a5297e82be419743be, store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:19,215 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bfed2856c31342439159341cb88ec430_f5c4bfa8bb8330a5297e82be419743be because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:19,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741979_1155 (size=12493) 2024-11-28T02:54:19,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741980_1156 (size=4469) 2024-11-28T02:54:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:19,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T02:54:19,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:19,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:19,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:19,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:19,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:19,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:19,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285414e4e1612a41a0bad02f396107c958_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762458709/Put/seqid=0 2024-11-28T02:54:19,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741981_1157 (size=14794) 2024-11-28T02:54:19,370 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:19,382 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285414e4e1612a41a0bad02f396107c958_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285414e4e1612a41a0bad02f396107c958_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:19,383 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/6ac2fcf5ae6c48f79026fabb7d7c88b1, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:19,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/6ac2fcf5ae6c48f79026fabb7d7c88b1 is 175, key is test_row_0/A:col10/1732762458709/Put/seqid=0 2024-11-28T02:54:19,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741982_1158 (size=39749) 2024-11-28T02:54:19,394 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/6ac2fcf5ae6c48f79026fabb7d7c88b1 2024-11-28T02:54:19,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/e5ce859f9b6e491693b786089371e8c9 is 50, key is test_row_0/B:col10/1732762458709/Put/seqid=0 2024-11-28T02:54:19,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762519416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:19,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741983_1159 (size=12151) 2024-11-28T02:54:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:19,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/e5ce859f9b6e491693b786089371e8c9 2024-11-28T02:54:19,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/8738b66afa394872b6a05ce4ee860e23 is 50, key is test_row_0/C:col10/1732762458709/Put/seqid=0 2024-11-28T02:54:19,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741984_1160 (size=12151) 2024-11-28T02:54:19,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T02:54:19,457 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-28T02:54:19,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762519536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:19,626 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/9ca89d38855f4bba89dc8f4e26491c2c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/9ca89d38855f4bba89dc8f4e26491c2c 2024-11-28T02:54:19,627 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#A#compaction#130 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:19,628 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/b8660cadbba14ed4b18e5c27d698a65a is 175, key is test_row_0/A:col10/1732762457481/Put/seqid=0 2024-11-28T02:54:19,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741985_1161 (size=31447) 2024-11-28T02:54:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T02:54:19,637 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:19,637 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:19,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-28T02:54:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T02:54:19,644 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:19,645 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:19,645 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/B of f5c4bfa8bb8330a5297e82be419743be into 9ca89d38855f4bba89dc8f4e26491c2c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:19,645 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:19,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:19,645 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/B, priority=12, startTime=1732762459173; duration=0sec 2024-11-28T02:54:19,645 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:19,645 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:B 2024-11-28T02:54:19,646 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:54:19,648 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:54:19,648 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/C is initiating minor compaction (all files) 2024-11-28T02:54:19,648 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/C in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:19,648 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/145709b5da844988ad28b8d9a5e2ba72, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/56ff90710f504e93a77636bbd64f0513, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/50bbe96ce9e843588b398fe3c926717f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/36563d1d1aa4407b98165ba152877940] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=47.5 K 2024-11-28T02:54:19,649 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 145709b5da844988ad28b8d9a5e2ba72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762455242 2024-11-28T02:54:19,651 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ff90710f504e93a77636bbd64f0513, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732762456148 2024-11-28T02:54:19,652 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 50bbe96ce9e843588b398fe3c926717f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732762456253 2024-11-28T02:54:19,652 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 36563d1d1aa4407b98165ba152877940, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732762457471 2024-11-28T02:54:19,664 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#C#compaction#134 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:19,665 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/eda84abd3aa547638f98175b84cdea0d is 50, key is test_row_0/C:col10/1732762457481/Put/seqid=0 2024-11-28T02:54:19,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741986_1162 (size=12493) 2024-11-28T02:54:19,694 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/eda84abd3aa547638f98175b84cdea0d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/eda84abd3aa547638f98175b84cdea0d 2024-11-28T02:54:19,708 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/C of f5c4bfa8bb8330a5297e82be419743be into eda84abd3aa547638f98175b84cdea0d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:19,708 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:19,708 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/C, priority=12, startTime=1732762459173; duration=0sec 2024-11-28T02:54:19,708 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:19,708 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:C 2024-11-28T02:54:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T02:54:19,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762519746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:19,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T02:54:19,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:19,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:19,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:19,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:19,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:19,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/8738b66afa394872b6a05ce4ee860e23 2024-11-28T02:54:19,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/6ac2fcf5ae6c48f79026fabb7d7c88b1 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/6ac2fcf5ae6c48f79026fabb7d7c88b1 2024-11-28T02:54:19,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/6ac2fcf5ae6c48f79026fabb7d7c88b1, entries=200, sequenceid=194, filesize=38.8 K 2024-11-28T02:54:19,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/e5ce859f9b6e491693b786089371e8c9 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/e5ce859f9b6e491693b786089371e8c9 2024-11-28T02:54:19,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/e5ce859f9b6e491693b786089371e8c9, entries=150, sequenceid=194, filesize=11.9 K 2024-11-28T02:54:19,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/8738b66afa394872b6a05ce4ee860e23 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/8738b66afa394872b6a05ce4ee860e23 2024-11-28T02:54:19,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/8738b66afa394872b6a05ce4ee860e23, entries=150, sequenceid=194, filesize=11.9 K 2024-11-28T02:54:19,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f5c4bfa8bb8330a5297e82be419743be in 535ms, sequenceid=194, compaction requested=false 2024-11-28T02:54:19,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:19,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T02:54:19,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:19,953 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T02:54:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:19,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T02:54:19,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128456f1c36e76846e1be985263c7765512_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762459412/Put/seqid=0 2024-11-28T02:54:19,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741987_1163 (size=12304) 2024-11-28T02:54:19,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:19,984 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128456f1c36e76846e1be985263c7765512_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128456f1c36e76846e1be985263c7765512_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:19,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/aeb3e536bf024d149a898f79b26b6f05, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:19,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/aeb3e536bf024d149a898f79b26b6f05 is 175, key is test_row_0/A:col10/1732762459412/Put/seqid=0 2024-11-28T02:54:19,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741988_1164 (size=31105) 2024-11-28T02:54:20,042 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/b8660cadbba14ed4b18e5c27d698a65a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/b8660cadbba14ed4b18e5c27d698a65a 2024-11-28T02:54:20,048 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/A of f5c4bfa8bb8330a5297e82be419743be into b8660cadbba14ed4b18e5c27d698a65a(size=30.7 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:20,048 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:20,048 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/A, priority=12, startTime=1732762459172; duration=0sec 2024-11-28T02:54:20,048 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:20,048 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:A 2024-11-28T02:54:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:20,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:20,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762520129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:20,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762520246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:20,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:20,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:20,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T02:54:20,395 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/aeb3e536bf024d149a898f79b26b6f05 2024-11-28T02:54:20,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/bf38414d033f4176b4f9a103dbf30158 is 50, key is test_row_0/B:col10/1732762459412/Put/seqid=0 2024-11-28T02:54:20,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741989_1165 (size=12151) 2024-11-28T02:54:20,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762520456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:20,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762520765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:20,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:20,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:20,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T02:54:20,807 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/bf38414d033f4176b4f9a103dbf30158 2024-11-28T02:54:20,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/b1b1000b38554027b2ff5ddc3a930eb7 is 50, key is test_row_0/C:col10/1732762459412/Put/seqid=0 2024-11-28T02:54:20,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741990_1166 (size=12151) 2024-11-28T02:54:21,222 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/b1b1000b38554027b2ff5ddc3a930eb7 2024-11-28T02:54:21,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/aeb3e536bf024d149a898f79b26b6f05 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/aeb3e536bf024d149a898f79b26b6f05 2024-11-28T02:54:21,231 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/aeb3e536bf024d149a898f79b26b6f05, entries=150, sequenceid=209, filesize=30.4 K 2024-11-28T02:54:21,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/bf38414d033f4176b4f9a103dbf30158 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/bf38414d033f4176b4f9a103dbf30158 2024-11-28T02:54:21,236 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/bf38414d033f4176b4f9a103dbf30158, entries=150, sequenceid=209, filesize=11.9 K 2024-11-28T02:54:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/b1b1000b38554027b2ff5ddc3a930eb7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/b1b1000b38554027b2ff5ddc3a930eb7 2024-11-28T02:54:21,243 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/b1b1000b38554027b2ff5ddc3a930eb7, entries=150, sequenceid=209, filesize=11.9 K 2024-11-28T02:54:21,244 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f5c4bfa8bb8330a5297e82be419743be in 1291ms, sequenceid=209, compaction requested=true 2024-11-28T02:54:21,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:21,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:21,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-28T02:54:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-28T02:54:21,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-28T02:54:21,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6010 sec 2024-11-28T02:54:21,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.6110 sec 2024-11-28T02:54:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:21,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T02:54:21,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:21,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:21,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:21,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:21,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:21,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:21,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128444c6afe02104822b61dcb37b5b3cb34_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762460123/Put/seqid=0 2024-11-28T02:54:21,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741991_1167 (size=14794) 2024-11-28T02:54:21,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762521348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:21,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762521456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:21,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762521666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:21,700 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:21,705 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128444c6afe02104822b61dcb37b5b3cb34_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128444c6afe02104822b61dcb37b5b3cb34_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:21,706 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/ab9ca15f90534265ad28331255349365, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:21,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/ab9ca15f90534265ad28331255349365 is 175, key is test_row_0/A:col10/1732762460123/Put/seqid=0 2024-11-28T02:54:21,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741992_1168 (size=39749) 2024-11-28T02:54:21,710 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/ab9ca15f90534265ad28331255349365 2024-11-28T02:54:21,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/c2a291aa93e046289c202856d48e41dc is 50, key is test_row_0/B:col10/1732762460123/Put/seqid=0 2024-11-28T02:54:21,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741993_1169 (size=12151) 2024-11-28T02:54:21,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39476 deadline: 1732762521736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:21,742 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:21,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39452 deadline: 1732762521745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:21,746 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_1' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:21,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:21,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39434 deadline: 1732762521766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:21,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1732762521766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:21,785 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:21,785 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T02:54:21,786 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:21,787 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-28T02:54:21,790 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T02:54:21,790 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:21,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T02:54:21,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T02:54:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:21,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:21,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762521976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:21,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:21,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:22,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T02:54:22,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:22,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:22,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:22,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:22,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T02:54:22,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/c2a291aa93e046289c202856d48e41dc 2024-11-28T02:54:22,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/04d910af10714157bc81bfb7900f1fb7 is 50, key is test_row_0/C:col10/1732762460123/Put/seqid=0 2024-11-28T02:54:22,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741994_1170 (size=12151) 2024-11-28T02:54:22,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/04d910af10714157bc81bfb7900f1fb7 2024-11-28T02:54:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/ab9ca15f90534265ad28331255349365 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/ab9ca15f90534265ad28331255349365 2024-11-28T02:54:22,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/ab9ca15f90534265ad28331255349365, entries=200, sequenceid=230, filesize=38.8 K 2024-11-28T02:54:22,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/c2a291aa93e046289c202856d48e41dc as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c2a291aa93e046289c202856d48e41dc 2024-11-28T02:54:22,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c2a291aa93e046289c202856d48e41dc, entries=150, sequenceid=230, filesize=11.9 K 2024-11-28T02:54:22,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/04d910af10714157bc81bfb7900f1fb7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/04d910af10714157bc81bfb7900f1fb7 2024-11-28T02:54:22,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/04d910af10714157bc81bfb7900f1fb7, entries=150, sequenceid=230, filesize=11.9 K 2024-11-28T02:54:22,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f5c4bfa8bb8330a5297e82be419743be in 903ms, sequenceid=230, compaction requested=true 2024-11-28T02:54:22,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:22,183 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:54:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:22,184 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:54:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:22,186 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:54:22,187 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/A is initiating minor compaction (all files) 2024-11-28T02:54:22,187 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/A in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:22,187 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:54:22,187 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/B is initiating minor compaction (all files) 2024-11-28T02:54:22,187 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/b8660cadbba14ed4b18e5c27d698a65a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/6ac2fcf5ae6c48f79026fabb7d7c88b1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/aeb3e536bf024d149a898f79b26b6f05, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/ab9ca15f90534265ad28331255349365] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=138.7 K 2024-11-28T02:54:22,187 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/B in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:22,187 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:22,187 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/9ca89d38855f4bba89dc8f4e26491c2c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/e5ce859f9b6e491693b786089371e8c9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/bf38414d033f4176b4f9a103dbf30158, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c2a291aa93e046289c202856d48e41dc] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=47.8 K 2024-11-28T02:54:22,187 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/b8660cadbba14ed4b18e5c27d698a65a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/6ac2fcf5ae6c48f79026fabb7d7c88b1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/aeb3e536bf024d149a898f79b26b6f05, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/ab9ca15f90534265ad28331255349365] 2024-11-28T02:54:22,187 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ca89d38855f4bba89dc8f4e26491c2c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732762457471 2024-11-28T02:54:22,187 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8660cadbba14ed4b18e5c27d698a65a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732762457471 2024-11-28T02:54:22,188 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ac2fcf5ae6c48f79026fabb7d7c88b1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732762458661 2024-11-28T02:54:22,188 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting e5ce859f9b6e491693b786089371e8c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732762458709 2024-11-28T02:54:22,188 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeb3e536bf024d149a898f79b26b6f05, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732762459393 2024-11-28T02:54:22,188 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting bf38414d033f4176b4f9a103dbf30158, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732762459393 2024-11-28T02:54:22,188 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab9ca15f90534265ad28331255349365, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732762460112 2024-11-28T02:54:22,188 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting c2a291aa93e046289c202856d48e41dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732762460123 2024-11-28T02:54:22,203 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:22,204 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#B#compaction#141 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:22,205 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/7d2d13bd5dc943fbb27e7e2ca762c360 is 50, key is test_row_0/B:col10/1732762460123/Put/seqid=0 2024-11-28T02:54:22,206 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128b9265fba107f4b7793001ebd2af6bb83_f5c4bfa8bb8330a5297e82be419743be store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:22,209 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128b9265fba107f4b7793001ebd2af6bb83_f5c4bfa8bb8330a5297e82be419743be, store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:22,209 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b9265fba107f4b7793001ebd2af6bb83_f5c4bfa8bb8330a5297e82be419743be because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:22,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741995_1171 (size=12629) 2024-11-28T02:54:22,219 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/7d2d13bd5dc943fbb27e7e2ca762c360 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/7d2d13bd5dc943fbb27e7e2ca762c360 2024-11-28T02:54:22,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741996_1172 (size=4469) 2024-11-28T02:54:22,221 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#A#compaction#142 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:22,222 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/d7a2360fc77f40829c2220af3f7bc0c2 is 175, key is test_row_0/A:col10/1732762460123/Put/seqid=0 2024-11-28T02:54:22,229 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/B of f5c4bfa8bb8330a5297e82be419743be into 7d2d13bd5dc943fbb27e7e2ca762c360(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:22,229 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:22,229 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/B, priority=12, startTime=1732762462184; duration=0sec 2024-11-28T02:54:22,229 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:22,229 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:B 2024-11-28T02:54:22,230 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:54:22,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741997_1173 (size=31583) 2024-11-28T02:54:22,231 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:54:22,231 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/C is initiating minor compaction (all files) 2024-11-28T02:54:22,231 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/C in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:22,232 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/eda84abd3aa547638f98175b84cdea0d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/8738b66afa394872b6a05ce4ee860e23, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/b1b1000b38554027b2ff5ddc3a930eb7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/04d910af10714157bc81bfb7900f1fb7] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=47.8 K 2024-11-28T02:54:22,233 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting eda84abd3aa547638f98175b84cdea0d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732762457471 2024-11-28T02:54:22,233 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8738b66afa394872b6a05ce4ee860e23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732762458709 2024-11-28T02:54:22,234 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting b1b1000b38554027b2ff5ddc3a930eb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732762459393 2024-11-28T02:54:22,234 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 04d910af10714157bc81bfb7900f1fb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732762460123 2024-11-28T02:54:22,246 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#C#compaction#143 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:22,247 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/98dc8027c8ae45ec96602342a59e59c1 is 50, key is test_row_0/C:col10/1732762460123/Put/seqid=0 2024-11-28T02:54:22,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T02:54:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:22,250 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T02:54:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:22,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741998_1174 (size=12629) 2024-11-28T02:54:22,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112837b6d36cebc64f32a000b826383dd759_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762461327/Put/seqid=0 2024-11-28T02:54:22,271 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/98dc8027c8ae45ec96602342a59e59c1 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/98dc8027c8ae45ec96602342a59e59c1 2024-11-28T02:54:22,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741999_1175 (size=12304) 2024-11-28T02:54:22,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:22,282 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/C of f5c4bfa8bb8330a5297e82be419743be into 98dc8027c8ae45ec96602342a59e59c1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:22,282 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:22,282 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/C, priority=12, startTime=1732762462184; duration=0sec 2024-11-28T02:54:22,282 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:22,282 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:C 2024-11-28T02:54:22,286 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112837b6d36cebc64f32a000b826383dd759_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112837b6d36cebc64f32a000b826383dd759_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:22,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/3887afd3f7e34e5d8feb4eea01f769ae, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:22,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/3887afd3f7e34e5d8feb4eea01f769ae is 175, key is test_row_0/A:col10/1732762461327/Put/seqid=0 2024-11-28T02:54:22,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742000_1176 (size=31105) 2024-11-28T02:54:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T02:54:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:22,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:22,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762522556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:22,633 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/d7a2360fc77f40829c2220af3f7bc0c2 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/d7a2360fc77f40829c2220af3f7bc0c2 2024-11-28T02:54:22,639 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/A of f5c4bfa8bb8330a5297e82be419743be into d7a2360fc77f40829c2220af3f7bc0c2(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:22,639 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:22,639 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/A, priority=12, startTime=1732762462183; duration=0sec 2024-11-28T02:54:22,639 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:22,639 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:A 2024-11-28T02:54:22,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762522666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:22,694 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/3887afd3f7e34e5d8feb4eea01f769ae 2024-11-28T02:54:22,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/697876e91f154fd68eacbb16bfee7e9f is 50, key is test_row_0/B:col10/1732762461327/Put/seqid=0 2024-11-28T02:54:22,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742001_1177 (size=12151) 2024-11-28T02:54:22,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762522876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T02:54:23,105 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/697876e91f154fd68eacbb16bfee7e9f 2024-11-28T02:54:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/757884889793488fabe60e967a701dfc is 50, key is test_row_0/C:col10/1732762461327/Put/seqid=0 2024-11-28T02:54:23,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742002_1178 (size=12151) 2024-11-28T02:54:23,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762523185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:23,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:23,525 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/757884889793488fabe60e967a701dfc 2024-11-28T02:54:23,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/3887afd3f7e34e5d8feb4eea01f769ae as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3887afd3f7e34e5d8feb4eea01f769ae 2024-11-28T02:54:23,534 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3887afd3f7e34e5d8feb4eea01f769ae, entries=150, sequenceid=243, filesize=30.4 K 2024-11-28T02:54:23,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/697876e91f154fd68eacbb16bfee7e9f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/697876e91f154fd68eacbb16bfee7e9f 2024-11-28T02:54:23,539 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/697876e91f154fd68eacbb16bfee7e9f, entries=150, sequenceid=243, filesize=11.9 K 2024-11-28T02:54:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/757884889793488fabe60e967a701dfc as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/757884889793488fabe60e967a701dfc 2024-11-28T02:54:23,545 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/757884889793488fabe60e967a701dfc, entries=150, sequenceid=243, filesize=11.9 K 2024-11-28T02:54:23,546 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f5c4bfa8bb8330a5297e82be419743be in 1296ms, sequenceid=243, compaction requested=false 2024-11-28T02:54:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-28T02:54:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-28T02:54:23,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-28T02:54:23,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7570 sec 2024-11-28T02:54:23,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.7630 sec 2024-11-28T02:54:23,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T02:54:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:23,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112884ecd5109e22426e949197129f79b5d3_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762462542/Put/seqid=0 2024-11-28T02:54:23,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742003_1179 (size=14994) 2024-11-28T02:54:23,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762523745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:23,810 INFO [master/978bd30859c3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-28T02:54:23,810 INFO [master/978bd30859c3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-28T02:54:23,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762523856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T02:54:23,936 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:23,937 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-28T02:54:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T02:54:23,938 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:23,939 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:23,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T02:54:24,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762524066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:24,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:24,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:24,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:24,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T02:54:24,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:24,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:24,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:24,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:24,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:24,125 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,130 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112884ecd5109e22426e949197129f79b5d3_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112884ecd5109e22426e949197129f79b5d3_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:24,131 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/4d8f6a39a2b44a5992f3ada870b81957, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:24,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/4d8f6a39a2b44a5992f3ada870b81957 is 175, key is test_row_0/A:col10/1732762462542/Put/seqid=0 2024-11-28T02:54:24,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742004_1180 (size=39949) 2024-11-28T02:54:24,137 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/4d8f6a39a2b44a5992f3ada870b81957 2024-11-28T02:54:24,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/fc038de5200a4416b080fc8ca6686861 is 50, key is test_row_0/B:col10/1732762462542/Put/seqid=0 2024-11-28T02:54:24,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742005_1181 (size=12301) 2024-11-28T02:54:24,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/fc038de5200a4416b080fc8ca6686861 2024-11-28T02:54:24,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/a18645d96db4413eabbad8967342fc61 is 50, key is test_row_0/C:col10/1732762462542/Put/seqid=0 2024-11-28T02:54:24,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742006_1182 (size=12301) 2024-11-28T02:54:24,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T02:54:24,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:24,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:24,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:24,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:24,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T02:54:24,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762524376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:24,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T02:54:24,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:24,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:24,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:24,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:24,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:24,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T02:54:24,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:24,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:24,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:24,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:24,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:24,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/a18645d96db4413eabbad8967342fc61 2024-11-28T02:54:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T02:54:24,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/4d8f6a39a2b44a5992f3ada870b81957 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/4d8f6a39a2b44a5992f3ada870b81957 2024-11-28T02:54:24,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/4d8f6a39a2b44a5992f3ada870b81957, entries=200, sequenceid=266, filesize=39.0 K 2024-11-28T02:54:24,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/fc038de5200a4416b080fc8ca6686861 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/fc038de5200a4416b080fc8ca6686861 2024-11-28T02:54:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/fc038de5200a4416b080fc8ca6686861, entries=150, sequenceid=266, filesize=12.0 K 2024-11-28T02:54:24,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/a18645d96db4413eabbad8967342fc61 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/a18645d96db4413eabbad8967342fc61 2024-11-28T02:54:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/a18645d96db4413eabbad8967342fc61, entries=150, sequenceid=266, filesize=12.0 K 2024-11-28T02:54:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for f5c4bfa8bb8330a5297e82be419743be in 888ms, sequenceid=266, compaction requested=true 2024-11-28T02:54:24,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:24,597 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:24,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:24,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:24,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,597 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:24,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,598 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102637 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:24,598 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/A is initiating minor compaction (all files) 2024-11-28T02:54:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,598 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/A in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:24,598 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/d7a2360fc77f40829c2220af3f7bc0c2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3887afd3f7e34e5d8feb4eea01f769ae, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/4d8f6a39a2b44a5992f3ada870b81957] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=100.2 K 2024-11-28T02:54:24,598 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:24,598 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/d7a2360fc77f40829c2220af3f7bc0c2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3887afd3f7e34e5d8feb4eea01f769ae, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/4d8f6a39a2b44a5992f3ada870b81957] 2024-11-28T02:54:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,598 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:24,599 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/B is initiating minor compaction (all files) 2024-11-28T02:54:24,599 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/B in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,599 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/7d2d13bd5dc943fbb27e7e2ca762c360, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/697876e91f154fd68eacbb16bfee7e9f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/fc038de5200a4416b080fc8ca6686861] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=36.2 K 2024-11-28T02:54:24,599 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7a2360fc77f40829c2220af3f7bc0c2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732762460123 2024-11-28T02:54:24,599 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d2d13bd5dc943fbb27e7e2ca762c360, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732762460123 2024-11-28T02:54:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,600 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3887afd3f7e34e5d8feb4eea01f769ae, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732762461327 2024-11-28T02:54:24,600 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 697876e91f154fd68eacbb16bfee7e9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732762461327 2024-11-28T02:54:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,600 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d8f6a39a2b44a5992f3ada870b81957, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732762462542 2024-11-28T02:54:24,600 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting fc038de5200a4416b080fc8ca6686861, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732762462542 2024-11-28T02:54:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,608 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,611 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128182ec3f353ae46a5aeb06d6dae3aae11_f5c4bfa8bb8330a5297e82be419743be store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:24,612 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#B#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:24,613 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/8f56973b3bf5402ca239788d11a1af2c is 50, key is test_row_0/B:col10/1732762462542/Put/seqid=0 2024-11-28T02:54:24,613 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128182ec3f353ae46a5aeb06d6dae3aae11_f5c4bfa8bb8330a5297e82be419743be, store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:24,613 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128182ec3f353ae46a5aeb06d6dae3aae11_f5c4bfa8bb8330a5297e82be419743be because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742007_1183 (size=12881) 2024-11-28T02:54:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742008_1184 (size=4469) 2024-11-28T02:54:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,648 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/8f56973b3bf5402ca239788d11a1af2c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/8f56973b3bf5402ca239788d11a1af2c 2024-11-28T02:54:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,648 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#A#compaction#150 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,649 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/a81e5fbd5f71453fa87acb07348427d6 is 175, key is test_row_0/A:col10/1732762462542/Put/seqid=0 2024-11-28T02:54:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,656 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/B of f5c4bfa8bb8330a5297e82be419743be into 8f56973b3bf5402ca239788d11a1af2c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:24,656 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,656 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/B, priority=13, startTime=1732762464597; duration=0sec 2024-11-28T02:54:24,657 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:24,657 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:B 2024-11-28T02:54:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,657 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:24,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742009_1185 (size=31835) 2024-11-28T02:54:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,658 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:24,658 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/C is initiating minor compaction (all files) 2024-11-28T02:54:24,658 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/C in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:24,659 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/98dc8027c8ae45ec96602342a59e59c1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/757884889793488fabe60e967a701dfc, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/a18645d96db4413eabbad8967342fc61] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=36.2 K 2024-11-28T02:54:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,659 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 98dc8027c8ae45ec96602342a59e59c1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732762460123 2024-11-28T02:54:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,660 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 757884889793488fabe60e967a701dfc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732762461327 2024-11-28T02:54:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,660 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting a18645d96db4413eabbad8967342fc61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732762462542 2024-11-28T02:54:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,664 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/a81e5fbd5f71453fa87acb07348427d6 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/a81e5fbd5f71453fa87acb07348427d6 2024-11-28T02:54:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,671 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#C#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,672 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/1873746f333d46c0810c522df5cc2d57 is 50, key is test_row_0/C:col10/1732762462542/Put/seqid=0 2024-11-28T02:54:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,674 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/A of f5c4bfa8bb8330a5297e82be419743be into a81e5fbd5f71453fa87acb07348427d6(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:24,674 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:24,674 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/A, priority=13, startTime=1732762464596; duration=0sec 2024-11-28T02:54:24,675 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:24,675 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:A 2024-11-28T02:54:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742010_1186 (size=12881) 2024-11-28T02:54:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T02:54:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:24,703 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:54:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:24,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:24,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128df260e233b1b410483b974f1b3ec9178_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762463744/Put/seqid=0 2024-11-28T02:54:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742011_1187 (size=9914) 2024-11-28T02:54:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:24,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:25,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762525003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:25,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:25,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762464985, max timestamp=1732762465001Num uniques 859; 2024-11-28T02:54:25,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762464985, max timestamp=1732762465001Num uniques 859; 2024-11-28T02:54:25,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762464985, max timestamp=1732762465001Num uniques 859; 2024-11-28T02:54:25,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:25,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:25,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T02:54:25,086 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/1873746f333d46c0810c522df5cc2d57 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/1873746f333d46c0810c522df5cc2d57 2024-11-28T02:54:25,091 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/C of f5c4bfa8bb8330a5297e82be419743be into 1873746f333d46c0810c522df5cc2d57(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:25,091 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:25,091 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/C, priority=13, startTime=1732762464597; duration=0sec 2024-11-28T02:54:25,091 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:25,091 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:C 2024-11-28T02:54:25,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:25,126 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128df260e233b1b410483b974f1b3ec9178_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128df260e233b1b410483b974f1b3ec9178_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:25,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/98cf431a009646afbc2dd6bda41e759e, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:25,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/98cf431a009646afbc2dd6bda41e759e is 175, key is test_row_0/A:col10/1732762463744/Put/seqid=0 2024-11-28T02:54:25,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742012_1188 (size=22561) 2024-11-28T02:54:25,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762525171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:25,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762525286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:25,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762525496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:25,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:25,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:25,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:25,537 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/98cf431a009646afbc2dd6bda41e759e 2024-11-28T02:54:25,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/23e2f643e633410c966a62060435a0dd is 50, key is test_row_0/B:col10/1732762463744/Put/seqid=0 2024-11-28T02:54:25,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742013_1189 (size=9857) 2024-11-28T02:54:25,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762525806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:25,960 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/23e2f643e633410c966a62060435a0dd 2024-11-28T02:54:25,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/baeed75d14ef4fadb8ea2496be847002 is 50, key is test_row_0/C:col10/1732762463744/Put/seqid=0 2024-11-28T02:54:25,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742014_1190 (size=9857) 2024-11-28T02:54:25,976 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/baeed75d14ef4fadb8ea2496be847002 2024-11-28T02:54:25,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/98cf431a009646afbc2dd6bda41e759e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/98cf431a009646afbc2dd6bda41e759e 2024-11-28T02:54:25,985 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/98cf431a009646afbc2dd6bda41e759e, entries=100, sequenceid=279, filesize=22.0 K 2024-11-28T02:54:25,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/23e2f643e633410c966a62060435a0dd as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/23e2f643e633410c966a62060435a0dd 2024-11-28T02:54:25,990 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/23e2f643e633410c966a62060435a0dd, entries=100, sequenceid=279, filesize=9.6 K 2024-11-28T02:54:25,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/baeed75d14ef4fadb8ea2496be847002 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/baeed75d14ef4fadb8ea2496be847002 2024-11-28T02:54:25,996 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/baeed75d14ef4fadb8ea2496be847002, entries=100, sequenceid=279, filesize=9.6 K 2024-11-28T02:54:25,997 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f5c4bfa8bb8330a5297e82be419743be in 1294ms, sequenceid=279, compaction requested=false 2024-11-28T02:54:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-28T02:54:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-28T02:54:26,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-28T02:54:26,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0590 sec 2024-11-28T02:54:26,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.0640 sec 2024-11-28T02:54:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T02:54:26,086 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:26,086 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-28T02:54:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T02:54:26,088 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:26,089 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:26,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T02:54:26,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T02:54:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:26,242 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-28T02:54:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:26,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f05d778066d14fcaa2b906f9d83b602a_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762465126/Put/seqid=0 2024-11-28T02:54:26,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742015_1191 (size=12454) 2024-11-28T02:54:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:26,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:26,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762526359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T02:54:26,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762526476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:26,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:26,660 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f05d778066d14fcaa2b906f9d83b602a_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f05d778066d14fcaa2b906f9d83b602a_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:26,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/da922f9eb8fd44c1853e3f828cdbe122, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:26,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/da922f9eb8fd44c1853e3f828cdbe122 is 175, key is test_row_0/A:col10/1732762465126/Put/seqid=0 2024-11-28T02:54:26,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742016_1192 (size=31255) 2024-11-28T02:54:26,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762526686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:26,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:26,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:26,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T02:54:26,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762526995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:27,066 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/da922f9eb8fd44c1853e3f828cdbe122 2024-11-28T02:54:27,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/6ffed6496c6d4ddcb6aa37c2c471b32f is 50, key is test_row_0/B:col10/1732762465126/Put/seqid=0 2024-11-28T02:54:27,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742017_1193 (size=12301) 2024-11-28T02:54:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T02:54:27,486 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/6ffed6496c6d4ddcb6aa37c2c471b32f 2024-11-28T02:54:27,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/fcee509cb9fd437094ba01ce01625f4b is 50, key is test_row_0/C:col10/1732762465126/Put/seqid=0 2024-11-28T02:54:27,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742018_1194 (size=12301) 2024-11-28T02:54:27,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762527506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:27,899 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/fcee509cb9fd437094ba01ce01625f4b 2024-11-28T02:54:27,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/da922f9eb8fd44c1853e3f828cdbe122 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/da922f9eb8fd44c1853e3f828cdbe122 2024-11-28T02:54:27,907 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/da922f9eb8fd44c1853e3f828cdbe122, entries=150, sequenceid=316, filesize=30.5 K 2024-11-28T02:54:27,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/6ffed6496c6d4ddcb6aa37c2c471b32f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/6ffed6496c6d4ddcb6aa37c2c471b32f 2024-11-28T02:54:27,911 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/6ffed6496c6d4ddcb6aa37c2c471b32f, entries=150, sequenceid=316, filesize=12.0 K 2024-11-28T02:54:27,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/fcee509cb9fd437094ba01ce01625f4b as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/fcee509cb9fd437094ba01ce01625f4b 2024-11-28T02:54:27,918 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/fcee509cb9fd437094ba01ce01625f4b, entries=150, sequenceid=316, filesize=12.0 K 2024-11-28T02:54:27,919 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for f5c4bfa8bb8330a5297e82be419743be in 1678ms, sequenceid=316, compaction requested=true 2024-11-28T02:54:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-28T02:54:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-28T02:54:27,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-28T02:54:27,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-11-28T02:54:27,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8370 sec 2024-11-28T02:54:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T02:54:28,236 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:28,236 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-28T02:54:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T02:54:28,272 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:28,273 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:28,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T02:54:28,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T02:54:28,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:28,427 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:54:28,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:28,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:28,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:28,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:28,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:28,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:28,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128056688b4535c43819379cee13e19d6cb_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762466356/Put/seqid=0 2024-11-28T02:54:28,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742019_1195 (size=12454) 2024-11-28T02:54:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:28,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T02:54:28,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762468560, max timestamp=1732762468586Num uniques 859; 2024-11-28T02:54:28,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:54:28,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762468560, max timestamp=1732762468586Num uniques 859; 2024-11-28T02:54:28,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:54:28,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762468560, max timestamp=1732762468586Num uniques 859; 2024-11-28T02:54:28,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:54:28,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762528657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:28,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762528766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:28,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:28,859 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128056688b4535c43819379cee13e19d6cb_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128056688b4535c43819379cee13e19d6cb_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:28,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e94e880ba66949a2a673ef07e6123daf, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:28,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e94e880ba66949a2a673ef07e6123daf is 175, key is test_row_0/A:col10/1732762466356/Put/seqid=0 2024-11-28T02:54:28,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742020_1196 (size=31255) 2024-11-28T02:54:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T02:54:28,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:28,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762528976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:29,264 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e94e880ba66949a2a673ef07e6123daf 2024-11-28T02:54:29,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/cd32e5f9dda94e278a22370082d181c0 is 50, key is test_row_0/B:col10/1732762466356/Put/seqid=0 2024-11-28T02:54:29,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742021_1197 (size=12301) 2024-11-28T02:54:29,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762529285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:29,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T02:54:29,675 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/cd32e5f9dda94e278a22370082d181c0 2024-11-28T02:54:29,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/329fbfa4953445459bc69718f2a5b32f is 50, key is test_row_0/C:col10/1732762466356/Put/seqid=0 2024-11-28T02:54:29,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742022_1198 (size=12301) 2024-11-28T02:54:29,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762529796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:30,090 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/329fbfa4953445459bc69718f2a5b32f 2024-11-28T02:54:30,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/e94e880ba66949a2a673ef07e6123daf as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e94e880ba66949a2a673ef07e6123daf 2024-11-28T02:54:30,100 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e94e880ba66949a2a673ef07e6123daf, entries=150, sequenceid=327, filesize=30.5 K 2024-11-28T02:54:30,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/cd32e5f9dda94e278a22370082d181c0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/cd32e5f9dda94e278a22370082d181c0 2024-11-28T02:54:30,105 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/cd32e5f9dda94e278a22370082d181c0, entries=150, sequenceid=327, filesize=12.0 K 2024-11-28T02:54:30,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/329fbfa4953445459bc69718f2a5b32f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/329fbfa4953445459bc69718f2a5b32f 2024-11-28T02:54:30,110 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/329fbfa4953445459bc69718f2a5b32f, entries=150, sequenceid=327, filesize=12.0 K 2024-11-28T02:54:30,111 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f5c4bfa8bb8330a5297e82be419743be in 1684ms, sequenceid=327, compaction requested=true 2024-11-28T02:54:30,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:30,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:30,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-28T02:54:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-28T02:54:30,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-28T02:54:30,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8400 sec 2024-11-28T02:54:30,115 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8780 sec 2024-11-28T02:54:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T02:54:30,386 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:30,386 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-28T02:54:30,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T02:54:30,388 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:30,389 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:30,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T02:54:30,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T02:54:30,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:30,541 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-28T02:54:30,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:30,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:30,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:30,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:30,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:30,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112894304269dded43c28870076093e5883d_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762468654/Put/seqid=0 2024-11-28T02:54:30,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742023_1199 (size=12454) 2024-11-28T02:54:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T02:54:30,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:30,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:30,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762530846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:30,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:30,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:30,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:30,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:30,959 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112894304269dded43c28870076093e5883d_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112894304269dded43c28870076093e5883d_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:30,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/db974d9dc567436886bc910bc063fa5a, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:30,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/db974d9dc567436886bc910bc063fa5a is 175, key is test_row_0/A:col10/1732762468654/Put/seqid=0 2024-11-28T02:54:30,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762530956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:30,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742024_1200 (size=31255) 2024-11-28T02:54:30,967 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/db974d9dc567436886bc910bc063fa5a 2024-11-28T02:54:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/13684a1f811a4332a9a5170bf61732bb is 50, key is test_row_0/B:col10/1732762468654/Put/seqid=0 2024-11-28T02:54:30,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742025_1201 (size=12301) 2024-11-28T02:54:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T02:54:31,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762531165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:31,380 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/13684a1f811a4332a9a5170bf61732bb 2024-11-28T02:54:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/6fa35980d29343c7ae2de929060f2cc3 is 50, key is test_row_0/C:col10/1732762468654/Put/seqid=0 2024-11-28T02:54:31,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742026_1202 (size=12301) 2024-11-28T02:54:31,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762531476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T02:54:31,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39476 deadline: 1732762531786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:31,786 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:31,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:31,792 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/6fa35980d29343c7ae2de929060f2cc3 2024-11-28T02:54:31,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/db974d9dc567436886bc910bc063fa5a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/db974d9dc567436886bc910bc063fa5a 2024-11-28T02:54:31,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39452 deadline: 1732762531796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:31,800 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/db974d9dc567436886bc910bc063fa5a, entries=150, sequenceid=362, filesize=30.5 K 2024-11-28T02:54:31,800 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_1' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18277 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:31,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/13684a1f811a4332a9a5170bf61732bb as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/13684a1f811a4332a9a5170bf61732bb 2024-11-28T02:54:31,806 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/13684a1f811a4332a9a5170bf61732bb, entries=150, sequenceid=362, filesize=12.0 K 2024-11-28T02:54:31,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/6fa35980d29343c7ae2de929060f2cc3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/6fa35980d29343c7ae2de929060f2cc3 2024-11-28T02:54:31,811 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/6fa35980d29343c7ae2de929060f2cc3, entries=150, sequenceid=362, filesize=12.0 K 2024-11-28T02:54:31,812 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for f5c4bfa8bb8330a5297e82be419743be in 1271ms, sequenceid=362, compaction requested=true 2024-11-28T02:54:31,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:31,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:31,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-28T02:54:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-28T02:54:31,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-28T02:54:31,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4240 sec 2024-11-28T02:54:31,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.4290 sec 2024-11-28T02:54:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:31,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T02:54:31,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:31,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:31,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:31,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:31,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:31,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:31,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e5d8048db3c34783b2cb78729a78531c_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762470813/Put/seqid=0 2024-11-28T02:54:31,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742027_1203 (size=14994) 2024-11-28T02:54:31,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:31,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39434 deadline: 1732762531892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:31,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1732762531898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:31,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762531986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:32,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39434 deadline: 1732762532006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:32,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1732762532006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:32,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39434 deadline: 1732762532216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:32,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1732762532216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:32,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:32,240 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:32,244 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e5d8048db3c34783b2cb78729a78531c_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e5d8048db3c34783b2cb78729a78531c_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:32,245 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/0c64d9796b4141b09d81a03d37843c6a, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:32,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/0c64d9796b4141b09d81a03d37843c6a is 175, key is test_row_0/A:col10/1732762470813/Put/seqid=0 2024-11-28T02:54:32,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742028_1204 (size=39949) 2024-11-28T02:54:32,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39434 deadline: 1732762532525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:32,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1732762532526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T02:54:32,536 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:32,537 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:32,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-28T02:54:32,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T02:54:32,538 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:32,539 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:32,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T02:54:32,664 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/0c64d9796b4141b09d81a03d37843c6a 2024-11-28T02:54:32,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/c7e6f3fa5d614ae5acc73d641822819f is 50, key is test_row_0/B:col10/1732762470813/Put/seqid=0 2024-11-28T02:54:32,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742029_1205 (size=12301) 2024-11-28T02:54:32,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T02:54:32,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:32,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:32,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:32,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:32,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:32,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T02:54:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:32,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:32,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:32,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T02:54:32,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T02:54:32,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:32,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:32,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:32,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:32,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:33,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39444 deadline: 1732762532996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:33,001 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in f5c4bfa8bb8330a5297e82be419743be of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:33,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:33,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:33,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:33,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39434 deadline: 1732762533036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:33,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39490 deadline: 1732762533046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:33,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:33,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:33,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:54:33,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/c7e6f3fa5d614ae5acc73d641822819f 2024-11-28T02:54:33,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/efd94ce87a7e417194b4bdb473fe7c69 is 50, key is test_row_0/C:col10/1732762470813/Put/seqid=0 2024-11-28T02:54:33,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742030_1206 (size=12301) 2024-11-28T02:54:33,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T02:54:33,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:33,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:33,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:33,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:33,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T02:54:33,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T02:54:33,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:33,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:33,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:33,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:33,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:33,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:33,336 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-11-28T02:54:33,336 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:33,336 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:33,336 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:33,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:33,341 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-11-28T02:54:33,341 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:33,341 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:33,341 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:33,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:33,342 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-11-28T02:54:33,342 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:33,342 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:33,342 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:33,342 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:33,346 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-11-28T02:54:33,346 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:33,346 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:33,346 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:33,346 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:33,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T02:54:33,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:33,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:33,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:33,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:33,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:33,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/efd94ce87a7e417194b4bdb473fe7c69 2024-11-28T02:54:33,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/0c64d9796b4141b09d81a03d37843c6a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/0c64d9796b4141b09d81a03d37843c6a 2024-11-28T02:54:33,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/0c64d9796b4141b09d81a03d37843c6a, entries=200, sequenceid=374, filesize=39.0 K 2024-11-28T02:54:33,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/c7e6f3fa5d614ae5acc73d641822819f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c7e6f3fa5d614ae5acc73d641822819f 2024-11-28T02:54:33,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c7e6f3fa5d614ae5acc73d641822819f, entries=150, sequenceid=374, filesize=12.0 K 2024-11-28T02:54:33,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/efd94ce87a7e417194b4bdb473fe7c69 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/efd94ce87a7e417194b4bdb473fe7c69 2024-11-28T02:54:33,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/efd94ce87a7e417194b4bdb473fe7c69, entries=150, sequenceid=374, filesize=12.0 K 2024-11-28T02:54:33,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f5c4bfa8bb8330a5297e82be419743be in 1699ms, sequenceid=374, compaction requested=true 2024-11-28T02:54:33,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:33,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:33,528 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-28T02:54:33,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:33,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:33,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:33,528 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-28T02:54:33,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5c4bfa8bb8330a5297e82be419743be:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:33,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:33,530 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 188110 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-28T02:54:33,530 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 71942 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-28T02:54:33,530 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/A is initiating minor compaction (all files) 2024-11-28T02:54:33,530 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/B is initiating minor compaction (all files) 2024-11-28T02:54:33,530 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/A in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:33,530 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/B in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:33,531 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/8f56973b3bf5402ca239788d11a1af2c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/23e2f643e633410c966a62060435a0dd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/6ffed6496c6d4ddcb6aa37c2c471b32f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/cd32e5f9dda94e278a22370082d181c0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/13684a1f811a4332a9a5170bf61732bb, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c7e6f3fa5d614ae5acc73d641822819f] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=70.3 K 2024-11-28T02:54:33,531 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/a81e5fbd5f71453fa87acb07348427d6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/98cf431a009646afbc2dd6bda41e759e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/da922f9eb8fd44c1853e3f828cdbe122, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e94e880ba66949a2a673ef07e6123daf, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/db974d9dc567436886bc910bc063fa5a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/0c64d9796b4141b09d81a03d37843c6a] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=183.7 K 2024-11-28T02:54:33,531 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:33,531 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/a81e5fbd5f71453fa87acb07348427d6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/98cf431a009646afbc2dd6bda41e759e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/da922f9eb8fd44c1853e3f828cdbe122, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e94e880ba66949a2a673ef07e6123daf, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/db974d9dc567436886bc910bc063fa5a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/0c64d9796b4141b09d81a03d37843c6a] 2024-11-28T02:54:33,531 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f56973b3bf5402ca239788d11a1af2c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732762462542 2024-11-28T02:54:33,531 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a81e5fbd5f71453fa87acb07348427d6, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732762462542 2024-11-28T02:54:33,531 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e2f643e633410c966a62060435a0dd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732762463721 2024-11-28T02:54:33,531 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98cf431a009646afbc2dd6bda41e759e, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732762463721 2024-11-28T02:54:33,532 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ffed6496c6d4ddcb6aa37c2c471b32f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732762465126 2024-11-28T02:54:33,532 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting da922f9eb8fd44c1853e3f828cdbe122, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732762465126 2024-11-28T02:54:33,532 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting cd32e5f9dda94e278a22370082d181c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732762466340 2024-11-28T02:54:33,532 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e94e880ba66949a2a673ef07e6123daf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732762466340 2024-11-28T02:54:33,532 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 13684a1f811a4332a9a5170bf61732bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732762468612 2024-11-28T02:54:33,532 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting db974d9dc567436886bc910bc063fa5a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732762468612 2024-11-28T02:54:33,532 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting c7e6f3fa5d614ae5acc73d641822819f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732762470813 2024-11-28T02:54:33,532 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c64d9796b4141b09d81a03d37843c6a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732762470813 2024-11-28T02:54:33,543 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:33,543 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#B#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:33,544 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/c3780cc4a3284b3ba2f6153c33c15656 is 50, key is test_row_0/B:col10/1732762470813/Put/seqid=0 2024-11-28T02:54:33,544 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112865d0c8ca2f41413fa8d1206d7f29142b_f5c4bfa8bb8330a5297e82be419743be store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:33,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742031_1207 (size=13085) 2024-11-28T02:54:33,563 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112865d0c8ca2f41413fa8d1206d7f29142b_f5c4bfa8bb8330a5297e82be419743be, store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:33,563 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112865d0c8ca2f41413fa8d1206d7f29142b_f5c4bfa8bb8330a5297e82be419743be because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:33,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742032_1208 (size=4469) 2024-11-28T02:54:33,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T02:54:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:33,612 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T02:54:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:33,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112891493b0d36204d0a8386e953846500d3_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762471856/Put/seqid=0 2024-11-28T02:54:33,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742033_1209 (size=12454) 2024-11-28T02:54:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T02:54:33,957 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/c3780cc4a3284b3ba2f6153c33c15656 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c3780cc4a3284b3ba2f6153c33c15656 2024-11-28T02:54:33,963 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/B of f5c4bfa8bb8330a5297e82be419743be into c3780cc4a3284b3ba2f6153c33c15656(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:33,963 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:33,963 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/B, priority=10, startTime=1732762473528; duration=0sec 2024-11-28T02:54:33,963 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:33,963 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:B 2024-11-28T02:54:33,964 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-28T02:54:33,966 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 71942 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-28T02:54:33,966 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): f5c4bfa8bb8330a5297e82be419743be/C is initiating minor compaction (all files) 2024-11-28T02:54:33,966 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5c4bfa8bb8330a5297e82be419743be/C in TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:33,966 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/1873746f333d46c0810c522df5cc2d57, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/baeed75d14ef4fadb8ea2496be847002, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/fcee509cb9fd437094ba01ce01625f4b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/329fbfa4953445459bc69718f2a5b32f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/6fa35980d29343c7ae2de929060f2cc3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/efd94ce87a7e417194b4bdb473fe7c69] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp, totalSize=70.3 K 2024-11-28T02:54:33,966 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1873746f333d46c0810c522df5cc2d57, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732762462542 2024-11-28T02:54:33,967 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting baeed75d14ef4fadb8ea2496be847002, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732762463721 2024-11-28T02:54:33,967 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting fcee509cb9fd437094ba01ce01625f4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732762465126 2024-11-28T02:54:33,967 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 329fbfa4953445459bc69718f2a5b32f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732762466340 2024-11-28T02:54:33,968 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fa35980d29343c7ae2de929060f2cc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732762468612 2024-11-28T02:54:33,968 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting efd94ce87a7e417194b4bdb473fe7c69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732762470813 2024-11-28T02:54:33,968 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#A#compaction#169 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:33,969 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/cb041abb3a3e45ebbd01238087fd224f is 175, key is test_row_0/A:col10/1732762470813/Put/seqid=0 2024-11-28T02:54:33,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742034_1210 (size=32039) 2024-11-28T02:54:33,981 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5c4bfa8bb8330a5297e82be419743be#C#compaction#171 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:33,982 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/957700e172e549c0a5a4268498340a48 is 50, key is test_row_0/C:col10/1732762470813/Put/seqid=0 2024-11-28T02:54:33,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742035_1211 (size=13085) 2024-11-28T02:54:34,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:34,030 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112891493b0d36204d0a8386e953846500d3_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112891493b0d36204d0a8386e953846500d3_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:34,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/2a33cef960e64543bd696f9ebcad6fd9, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:34,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/2a33cef960e64543bd696f9ebcad6fd9 is 175, key is test_row_0/A:col10/1732762471856/Put/seqid=0 2024-11-28T02:54:34,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742036_1212 (size=31255) 2024-11-28T02:54:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:34,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. as already flushing 2024-11-28T02:54:34,057 INFO [Thread-586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-586. 2024-11-28T02:54:34,058 DEBUG [Thread-586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:34,058 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:34,058 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:34,058 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:34,067 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-11-28T02:54:34,067 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:34,067 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:34,067 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:34,067 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:34,383 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/cb041abb3a3e45ebbd01238087fd224f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/cb041abb3a3e45ebbd01238087fd224f 2024-11-28T02:54:34,391 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/A of f5c4bfa8bb8330a5297e82be419743be into cb041abb3a3e45ebbd01238087fd224f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:34,391 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:34,391 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/A, priority=10, startTime=1732762473528; duration=0sec 2024-11-28T02:54:34,391 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:34,391 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:A 2024-11-28T02:54:34,393 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/957700e172e549c0a5a4268498340a48 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/957700e172e549c0a5a4268498340a48 2024-11-28T02:54:34,398 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in f5c4bfa8bb8330a5297e82be419743be/C of f5c4bfa8bb8330a5297e82be419743be into 957700e172e549c0a5a4268498340a48(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:34,399 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:34,399 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., storeName=f5c4bfa8bb8330a5297e82be419743be/C, priority=10, startTime=1732762473528; duration=0sec 2024-11-28T02:54:34,399 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:34,399 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5c4bfa8bb8330a5297e82be419743be:C 2024-11-28T02:54:34,436 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/2a33cef960e64543bd696f9ebcad6fd9 2024-11-28T02:54:34,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/f55e92aae1e34f5c89547ba3f4a45010 is 50, key is test_row_0/B:col10/1732762471856/Put/seqid=0 2024-11-28T02:54:34,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742037_1213 (size=12301) 2024-11-28T02:54:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T02:54:34,855 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/f55e92aae1e34f5c89547ba3f4a45010 2024-11-28T02:54:34,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/06053ffa438e4e7b8dfa652925b2549c is 50, key is test_row_0/C:col10/1732762471856/Put/seqid=0 2024-11-28T02:54:34,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742038_1214 (size=12301) 2024-11-28T02:54:35,033 INFO [Thread-592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-592. 2024-11-28T02:54:35,033 DEBUG [Thread-592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:35,033 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:35,033 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:35,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:35,272 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/06053ffa438e4e7b8dfa652925b2549c 2024-11-28T02:54:35,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/2a33cef960e64543bd696f9ebcad6fd9 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/2a33cef960e64543bd696f9ebcad6fd9 2024-11-28T02:54:35,289 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/2a33cef960e64543bd696f9ebcad6fd9, entries=150, sequenceid=394, filesize=30.5 K 2024-11-28T02:54:35,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/f55e92aae1e34f5c89547ba3f4a45010 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/f55e92aae1e34f5c89547ba3f4a45010 2024-11-28T02:54:35,294 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/f55e92aae1e34f5c89547ba3f4a45010, entries=150, sequenceid=394, filesize=12.0 K 2024-11-28T02:54:35,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/06053ffa438e4e7b8dfa652925b2549c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/06053ffa438e4e7b8dfa652925b2549c 2024-11-28T02:54:35,297 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/06053ffa438e4e7b8dfa652925b2549c, entries=150, sequenceid=394, filesize=12.0 K 2024-11-28T02:54:35,298 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=20.13 KB/20610 for f5c4bfa8bb8330a5297e82be419743be in 1686ms, sequenceid=394, compaction requested=false 2024-11-28T02:54:35,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for f5c4bfa8bb8330a5297e82be419743be: 2024-11-28T02:54:35,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:35,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-28T02:54:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-28T02:54:35,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-28T02:54:35,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7610 sec 2024-11-28T02:54:35,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.7640 sec 2024-11-28T02:54:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T02:54:36,707 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:37,996 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T02:54:41,821 INFO [Thread-594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-594. 2024-11-28T02:54:41,821 DEBUG [Thread-594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:41,821 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:41,821 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:41,821 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:41,868 INFO [Thread-588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-588. 2024-11-28T02:54:41,868 DEBUG [Thread-588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:54:41,868 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:41,868 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:41,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 267 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2833 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2954 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1214 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3641 rows 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1217 2024-11-28T02:54:41,869 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3651 rows 2024-11-28T02:54:41,869 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:54:41,869 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:54:41,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:41,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:41,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:54:41,870 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T02:54:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T02:54:41,876 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762481875"}]},"ts":"1732762481875"} 2024-11-28T02:54:41,878 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T02:54:41,878 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T02:54:41,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T02:54:41,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, UNASSIGN}] 2024-11-28T02:54:41,881 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, UNASSIGN 2024-11-28T02:54:41,882 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=f5c4bfa8bb8330a5297e82be419743be, regionState=CLOSING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:54:41,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, UNASSIGN because future has completed 2024-11-28T02:54:41,884 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T02:54:41,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:54:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T02:54:42,038 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:42,038 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T02:54:42,039 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing f5c4bfa8bb8330a5297e82be419743be, disabling compactions & flushes 2024-11-28T02:54:42,039 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:42,039 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:42,039 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. after waiting 0 ms 2024-11-28T02:54:42,039 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:42,039 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing f5c4bfa8bb8330a5297e82be419743be 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T02:54:42,040 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=A 2024-11-28T02:54:42,040 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:42,040 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=B 2024-11-28T02:54:42,040 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:42,040 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5c4bfa8bb8330a5297e82be419743be, store=C 2024-11-28T02:54:42,041 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:42,049 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128edbcc3854cd54bbb96e9bdced74a3300_f5c4bfa8bb8330a5297e82be419743be is 50, key is test_row_0/A:col10/1732762475016/Put/seqid=0 2024-11-28T02:54:42,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742039_1215 (size=12454) 2024-11-28T02:54:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T02:54:42,454 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:42,462 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128edbcc3854cd54bbb96e9bdced74a3300_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128edbcc3854cd54bbb96e9bdced74a3300_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:42,464 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/1d4987fa8b844a92b945a8f202330918, store: [table=TestAcidGuarantees family=A region=f5c4bfa8bb8330a5297e82be419743be] 2024-11-28T02:54:42,465 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/1d4987fa8b844a92b945a8f202330918 is 175, key is test_row_0/A:col10/1732762475016/Put/seqid=0 2024-11-28T02:54:42,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742040_1216 (size=31255) 2024-11-28T02:54:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T02:54:42,870 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=405, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/1d4987fa8b844a92b945a8f202330918 2024-11-28T02:54:42,877 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/9b4c4a7afab84d6e9f4692cfd37d0728 is 50, key is test_row_0/B:col10/1732762475016/Put/seqid=0 2024-11-28T02:54:42,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742041_1217 (size=12301) 2024-11-28T02:54:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T02:54:43,282 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/9b4c4a7afab84d6e9f4692cfd37d0728 2024-11-28T02:54:43,297 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/4a4965de2bbc461295be8e929bb5e524 is 50, key is test_row_0/C:col10/1732762475016/Put/seqid=0 2024-11-28T02:54:43,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742042_1218 (size=12301) 2024-11-28T02:54:43,703 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/4a4965de2bbc461295be8e929bb5e524 2024-11-28T02:54:43,743 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/A/1d4987fa8b844a92b945a8f202330918 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/1d4987fa8b844a92b945a8f202330918 2024-11-28T02:54:43,751 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/1d4987fa8b844a92b945a8f202330918, entries=150, sequenceid=405, filesize=30.5 K 2024-11-28T02:54:43,752 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/B/9b4c4a7afab84d6e9f4692cfd37d0728 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/9b4c4a7afab84d6e9f4692cfd37d0728 2024-11-28T02:54:43,758 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/9b4c4a7afab84d6e9f4692cfd37d0728, entries=150, sequenceid=405, filesize=12.0 K 2024-11-28T02:54:43,759 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/.tmp/C/4a4965de2bbc461295be8e929bb5e524 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4a4965de2bbc461295be8e929bb5e524 2024-11-28T02:54:43,765 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4a4965de2bbc461295be8e929bb5e524, entries=150, sequenceid=405, filesize=12.0 K 2024-11-28T02:54:43,766 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f5c4bfa8bb8330a5297e82be419743be in 1727ms, sequenceid=405, compaction requested=true 2024-11-28T02:54:43,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/45828bd44ca94de3be3e557d50cb8265, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/60904d708f88473d878df59f370e4d17, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/0ae1628250f8465c97008d3b50dae6a2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/b4c7af5dad7940f3836c7294e626f6d4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e0bd4a20d91648a3a74d60040b14b0f4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/9c9928e2359d46f5902f62d56b2422b5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/8f36c9ad16394f41b261f1ea2b9ae53e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e8be2cd7531449e682ad86319ebbc8a3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e106156a114d42bc93f216a86a8e4171, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3255f0947e9240349b3ada0097377172, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/b8660cadbba14ed4b18e5c27d698a65a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/6ac2fcf5ae6c48f79026fabb7d7c88b1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/aeb3e536bf024d149a898f79b26b6f05, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/ab9ca15f90534265ad28331255349365, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/d7a2360fc77f40829c2220af3f7bc0c2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3887afd3f7e34e5d8feb4eea01f769ae, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/4d8f6a39a2b44a5992f3ada870b81957, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/a81e5fbd5f71453fa87acb07348427d6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/98cf431a009646afbc2dd6bda41e759e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/da922f9eb8fd44c1853e3f828cdbe122, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e94e880ba66949a2a673ef07e6123daf, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/db974d9dc567436886bc910bc063fa5a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/0c64d9796b4141b09d81a03d37843c6a] to archive 2024-11-28T02:54:43,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:54:43,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/45828bd44ca94de3be3e557d50cb8265 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/45828bd44ca94de3be3e557d50cb8265 2024-11-28T02:54:43,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/60904d708f88473d878df59f370e4d17 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/60904d708f88473d878df59f370e4d17 2024-11-28T02:54:43,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/0ae1628250f8465c97008d3b50dae6a2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/0ae1628250f8465c97008d3b50dae6a2 2024-11-28T02:54:43,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/b4c7af5dad7940f3836c7294e626f6d4 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/b4c7af5dad7940f3836c7294e626f6d4 2024-11-28T02:54:43,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e0bd4a20d91648a3a74d60040b14b0f4 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e0bd4a20d91648a3a74d60040b14b0f4 2024-11-28T02:54:43,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/9c9928e2359d46f5902f62d56b2422b5 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/9c9928e2359d46f5902f62d56b2422b5 2024-11-28T02:54:43,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/8f36c9ad16394f41b261f1ea2b9ae53e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/8f36c9ad16394f41b261f1ea2b9ae53e 2024-11-28T02:54:43,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e8be2cd7531449e682ad86319ebbc8a3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e8be2cd7531449e682ad86319ebbc8a3 2024-11-28T02:54:43,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e106156a114d42bc93f216a86a8e4171 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e106156a114d42bc93f216a86a8e4171 2024-11-28T02:54:43,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3255f0947e9240349b3ada0097377172 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3255f0947e9240349b3ada0097377172 2024-11-28T02:54:43,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/b8660cadbba14ed4b18e5c27d698a65a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/b8660cadbba14ed4b18e5c27d698a65a 2024-11-28T02:54:43,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/6ac2fcf5ae6c48f79026fabb7d7c88b1 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/6ac2fcf5ae6c48f79026fabb7d7c88b1 2024-11-28T02:54:43,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/aeb3e536bf024d149a898f79b26b6f05 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/aeb3e536bf024d149a898f79b26b6f05 2024-11-28T02:54:43,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/ab9ca15f90534265ad28331255349365 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/ab9ca15f90534265ad28331255349365 2024-11-28T02:54:43,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/d7a2360fc77f40829c2220af3f7bc0c2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/d7a2360fc77f40829c2220af3f7bc0c2 2024-11-28T02:54:43,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3887afd3f7e34e5d8feb4eea01f769ae to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/3887afd3f7e34e5d8feb4eea01f769ae 2024-11-28T02:54:43,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/4d8f6a39a2b44a5992f3ada870b81957 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/4d8f6a39a2b44a5992f3ada870b81957 2024-11-28T02:54:43,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/a81e5fbd5f71453fa87acb07348427d6 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/a81e5fbd5f71453fa87acb07348427d6 2024-11-28T02:54:43,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/98cf431a009646afbc2dd6bda41e759e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/98cf431a009646afbc2dd6bda41e759e 2024-11-28T02:54:43,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/da922f9eb8fd44c1853e3f828cdbe122 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/da922f9eb8fd44c1853e3f828cdbe122 2024-11-28T02:54:43,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e94e880ba66949a2a673ef07e6123daf to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/e94e880ba66949a2a673ef07e6123daf 2024-11-28T02:54:43,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/db974d9dc567436886bc910bc063fa5a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/db974d9dc567436886bc910bc063fa5a 2024-11-28T02:54:43,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/0c64d9796b4141b09d81a03d37843c6a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/0c64d9796b4141b09d81a03d37843c6a 2024-11-28T02:54:43,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/1ab40fd58b42460a99c6cc53d974599c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/0c77e237756c45e7a383062d59798c11, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/6bed607304184e8386213ed2814fcdfc, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/e355bc209020404d94f0792df72a6773, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/f4df4bf26c2047899ffebd8ed143019f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/337a56282f4e43e9bfb2ceb77db13da9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/da9d5788c25f4b6599c0fab6e0330c81, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/2b6382263589406e9bcdd26d4a76c1af, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/fe083d19e0154189a2e9a1e542ea4581, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/9ca89d38855f4bba89dc8f4e26491c2c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/3ce6433a30f94b3d991d2ce1c77beb72, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/e5ce859f9b6e491693b786089371e8c9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/bf38414d033f4176b4f9a103dbf30158, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/7d2d13bd5dc943fbb27e7e2ca762c360, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c2a291aa93e046289c202856d48e41dc, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/697876e91f154fd68eacbb16bfee7e9f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/8f56973b3bf5402ca239788d11a1af2c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/fc038de5200a4416b080fc8ca6686861, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/23e2f643e633410c966a62060435a0dd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/6ffed6496c6d4ddcb6aa37c2c471b32f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/cd32e5f9dda94e278a22370082d181c0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/13684a1f811a4332a9a5170bf61732bb, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c7e6f3fa5d614ae5acc73d641822819f] to archive 2024-11-28T02:54:43,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:54:43,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/1ab40fd58b42460a99c6cc53d974599c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/1ab40fd58b42460a99c6cc53d974599c 2024-11-28T02:54:43,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/0c77e237756c45e7a383062d59798c11 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/0c77e237756c45e7a383062d59798c11 2024-11-28T02:54:43,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/6bed607304184e8386213ed2814fcdfc to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/6bed607304184e8386213ed2814fcdfc 2024-11-28T02:54:43,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/e355bc209020404d94f0792df72a6773 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/e355bc209020404d94f0792df72a6773 2024-11-28T02:54:43,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/f4df4bf26c2047899ffebd8ed143019f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/f4df4bf26c2047899ffebd8ed143019f 2024-11-28T02:54:43,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/337a56282f4e43e9bfb2ceb77db13da9 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/337a56282f4e43e9bfb2ceb77db13da9 2024-11-28T02:54:43,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/da9d5788c25f4b6599c0fab6e0330c81 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/da9d5788c25f4b6599c0fab6e0330c81 2024-11-28T02:54:43,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/2b6382263589406e9bcdd26d4a76c1af to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/2b6382263589406e9bcdd26d4a76c1af 2024-11-28T02:54:43,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/fe083d19e0154189a2e9a1e542ea4581 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/fe083d19e0154189a2e9a1e542ea4581 2024-11-28T02:54:43,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/9ca89d38855f4bba89dc8f4e26491c2c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/9ca89d38855f4bba89dc8f4e26491c2c 2024-11-28T02:54:43,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/3ce6433a30f94b3d991d2ce1c77beb72 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/3ce6433a30f94b3d991d2ce1c77beb72 2024-11-28T02:54:43,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/e5ce859f9b6e491693b786089371e8c9 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/e5ce859f9b6e491693b786089371e8c9 2024-11-28T02:54:43,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/bf38414d033f4176b4f9a103dbf30158 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/bf38414d033f4176b4f9a103dbf30158 2024-11-28T02:54:43,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/7d2d13bd5dc943fbb27e7e2ca762c360 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/7d2d13bd5dc943fbb27e7e2ca762c360 2024-11-28T02:54:43,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c2a291aa93e046289c202856d48e41dc to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c2a291aa93e046289c202856d48e41dc 2024-11-28T02:54:43,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/697876e91f154fd68eacbb16bfee7e9f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/697876e91f154fd68eacbb16bfee7e9f 2024-11-28T02:54:43,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/8f56973b3bf5402ca239788d11a1af2c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/8f56973b3bf5402ca239788d11a1af2c 2024-11-28T02:54:43,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/fc038de5200a4416b080fc8ca6686861 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/fc038de5200a4416b080fc8ca6686861 2024-11-28T02:54:43,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/23e2f643e633410c966a62060435a0dd to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/23e2f643e633410c966a62060435a0dd 2024-11-28T02:54:43,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/6ffed6496c6d4ddcb6aa37c2c471b32f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/6ffed6496c6d4ddcb6aa37c2c471b32f 2024-11-28T02:54:43,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/cd32e5f9dda94e278a22370082d181c0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/cd32e5f9dda94e278a22370082d181c0 2024-11-28T02:54:43,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/13684a1f811a4332a9a5170bf61732bb to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/13684a1f811a4332a9a5170bf61732bb 2024-11-28T02:54:43,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c7e6f3fa5d614ae5acc73d641822819f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c7e6f3fa5d614ae5acc73d641822819f 2024-11-28T02:54:43,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/2e22172147e34287b2a9b5e469644bf8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4aeec579b9cf456685a0cb1faea34be7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/44102efe408a4f4381cc5e428dbfb15c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/2c2b3d29610c43dc951a23cfebaf5a9a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4b4427953a3145d38af7ba40b54ee3b0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/145709b5da844988ad28b8d9a5e2ba72, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/6625f1bf8ac74ec78326212ed68c58cb, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/56ff90710f504e93a77636bbd64f0513, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/50bbe96ce9e843588b398fe3c926717f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/eda84abd3aa547638f98175b84cdea0d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/36563d1d1aa4407b98165ba152877940, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/8738b66afa394872b6a05ce4ee860e23, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/b1b1000b38554027b2ff5ddc3a930eb7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/98dc8027c8ae45ec96602342a59e59c1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/04d910af10714157bc81bfb7900f1fb7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/757884889793488fabe60e967a701dfc, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/1873746f333d46c0810c522df5cc2d57, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/a18645d96db4413eabbad8967342fc61, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/baeed75d14ef4fadb8ea2496be847002, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/fcee509cb9fd437094ba01ce01625f4b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/329fbfa4953445459bc69718f2a5b32f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/6fa35980d29343c7ae2de929060f2cc3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/efd94ce87a7e417194b4bdb473fe7c69] to archive 2024-11-28T02:54:43,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:54:43,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/2e22172147e34287b2a9b5e469644bf8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/2e22172147e34287b2a9b5e469644bf8 2024-11-28T02:54:43,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4aeec579b9cf456685a0cb1faea34be7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4aeec579b9cf456685a0cb1faea34be7 2024-11-28T02:54:43,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/44102efe408a4f4381cc5e428dbfb15c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/44102efe408a4f4381cc5e428dbfb15c 2024-11-28T02:54:43,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/2c2b3d29610c43dc951a23cfebaf5a9a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/2c2b3d29610c43dc951a23cfebaf5a9a 2024-11-28T02:54:43,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4b4427953a3145d38af7ba40b54ee3b0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4b4427953a3145d38af7ba40b54ee3b0 2024-11-28T02:54:43,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/145709b5da844988ad28b8d9a5e2ba72 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/145709b5da844988ad28b8d9a5e2ba72 2024-11-28T02:54:43,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/6625f1bf8ac74ec78326212ed68c58cb to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/6625f1bf8ac74ec78326212ed68c58cb 2024-11-28T02:54:43,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/56ff90710f504e93a77636bbd64f0513 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/56ff90710f504e93a77636bbd64f0513 2024-11-28T02:54:43,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/50bbe96ce9e843588b398fe3c926717f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/50bbe96ce9e843588b398fe3c926717f 2024-11-28T02:54:43,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/eda84abd3aa547638f98175b84cdea0d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/eda84abd3aa547638f98175b84cdea0d 2024-11-28T02:54:43,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/36563d1d1aa4407b98165ba152877940 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/36563d1d1aa4407b98165ba152877940 2024-11-28T02:54:43,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/8738b66afa394872b6a05ce4ee860e23 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/8738b66afa394872b6a05ce4ee860e23 2024-11-28T02:54:43,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/b1b1000b38554027b2ff5ddc3a930eb7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/b1b1000b38554027b2ff5ddc3a930eb7 2024-11-28T02:54:43,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/98dc8027c8ae45ec96602342a59e59c1 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/98dc8027c8ae45ec96602342a59e59c1 2024-11-28T02:54:43,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/04d910af10714157bc81bfb7900f1fb7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/04d910af10714157bc81bfb7900f1fb7 2024-11-28T02:54:43,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/757884889793488fabe60e967a701dfc to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/757884889793488fabe60e967a701dfc 2024-11-28T02:54:43,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/1873746f333d46c0810c522df5cc2d57 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/1873746f333d46c0810c522df5cc2d57 2024-11-28T02:54:43,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/a18645d96db4413eabbad8967342fc61 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/a18645d96db4413eabbad8967342fc61 2024-11-28T02:54:43,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/baeed75d14ef4fadb8ea2496be847002 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/baeed75d14ef4fadb8ea2496be847002 2024-11-28T02:54:43,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/fcee509cb9fd437094ba01ce01625f4b to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/fcee509cb9fd437094ba01ce01625f4b 2024-11-28T02:54:43,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/329fbfa4953445459bc69718f2a5b32f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/329fbfa4953445459bc69718f2a5b32f 2024-11-28T02:54:43,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/6fa35980d29343c7ae2de929060f2cc3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/6fa35980d29343c7ae2de929060f2cc3 2024-11-28T02:54:43,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/efd94ce87a7e417194b4bdb473fe7c69 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/efd94ce87a7e417194b4bdb473fe7c69 2024-11-28T02:54:43,838 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/recovered.edits/408.seqid, newMaxSeqId=408, maxSeqId=4 2024-11-28T02:54:43,839 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. 2024-11-28T02:54:43,839 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for f5c4bfa8bb8330a5297e82be419743be: Waiting for close lock at 1732762482038Running coprocessor pre-close hooks at 1732762482038Disabling compacts and flushes for region at 1732762482038Disabling writes for close at 1732762482039 (+1 ms)Obtaining lock to block concurrent updates at 1732762482039Preparing flush snapshotting stores in f5c4bfa8bb8330a5297e82be419743be at 1732762482039Finished memstore snapshotting TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732762482041 (+2 ms)Flushing stores of TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be. at 1732762482042 (+1 ms)Flushing f5c4bfa8bb8330a5297e82be419743be/A: creating writer at 1732762482042Flushing mob file f5c4bfa8bb8330a5297e82be419743be/A: appending metadata at 1732762482049 (+7 ms)Flushing mob file f5c4bfa8bb8330a5297e82be419743be/A: closing flushed file at 1732762482049Flushing f5c4bfa8bb8330a5297e82be419743be/A: appending metadata at 1732762482464 (+415 ms)Flushing f5c4bfa8bb8330a5297e82be419743be/A: closing flushed file at 1732762482464Flushing f5c4bfa8bb8330a5297e82be419743be/B: creating writer at 1732762482874 (+410 ms)Flushing f5c4bfa8bb8330a5297e82be419743be/B: appending metadata at 1732762482877 (+3 ms)Flushing f5c4bfa8bb8330a5297e82be419743be/B: closing flushed file at 1732762482877Flushing f5c4bfa8bb8330a5297e82be419743be/C: creating writer at 1732762483293 (+416 ms)Flushing f5c4bfa8bb8330a5297e82be419743be/C: appending metadata at 1732762483297 (+4 ms)Flushing f5c4bfa8bb8330a5297e82be419743be/C: closing flushed file at 1732762483297Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b533161: reopening flushed file at 1732762483741 (+444 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9b678af: reopening flushed file at 1732762483751 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60bfd391: reopening flushed file at 1732762483758 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f5c4bfa8bb8330a5297e82be419743be in 1727ms, sequenceid=405, compaction requested=true at 1732762483766 (+8 ms)Writing region close event to WAL at 1732762483835 (+69 ms)Running coprocessor post-close hooks at 1732762483838 (+3 ms)Closed at 1732762483839 (+1 ms) 2024-11-28T02:54:43,841 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:43,842 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=f5c4bfa8bb8330a5297e82be419743be, regionState=CLOSED 2024-11-28T02:54:43,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:54:43,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-28T02:54:43,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure f5c4bfa8bb8330a5297e82be419743be, server=978bd30859c3,41631,1732762419946 in 1.9600 sec 2024-11-28T02:54:43,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-28T02:54:43,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5c4bfa8bb8330a5297e82be419743be, UNASSIGN in 1.9650 sec 2024-11-28T02:54:43,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-28T02:54:43,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9680 sec 2024-11-28T02:54:43,850 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762483850"}]},"ts":"1732762483850"} 2024-11-28T02:54:43,851 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T02:54:43,851 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T02:54:43,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9800 sec 2024-11-28T02:54:44,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T02:54:44,028 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:44,031 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T02:54:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:44,035 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T02:54:44,036 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:44,039 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,043 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/recovered.edits] 2024-11-28T02:54:44,046 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/1d4987fa8b844a92b945a8f202330918 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/1d4987fa8b844a92b945a8f202330918 2024-11-28T02:54:44,047 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/2a33cef960e64543bd696f9ebcad6fd9 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/2a33cef960e64543bd696f9ebcad6fd9 2024-11-28T02:54:44,048 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/cb041abb3a3e45ebbd01238087fd224f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/A/cb041abb3a3e45ebbd01238087fd224f 2024-11-28T02:54:44,051 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/9b4c4a7afab84d6e9f4692cfd37d0728 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/9b4c4a7afab84d6e9f4692cfd37d0728 2024-11-28T02:54:44,052 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c3780cc4a3284b3ba2f6153c33c15656 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/c3780cc4a3284b3ba2f6153c33c15656 2024-11-28T02:54:44,053 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/f55e92aae1e34f5c89547ba3f4a45010 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/B/f55e92aae1e34f5c89547ba3f4a45010 2024-11-28T02:54:44,055 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/06053ffa438e4e7b8dfa652925b2549c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/06053ffa438e4e7b8dfa652925b2549c 2024-11-28T02:54:44,056 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4a4965de2bbc461295be8e929bb5e524 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/4a4965de2bbc461295be8e929bb5e524 2024-11-28T02:54:44,057 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/957700e172e549c0a5a4268498340a48 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/C/957700e172e549c0a5a4268498340a48 2024-11-28T02:54:44,059 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/recovered.edits/408.seqid to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be/recovered.edits/408.seqid 2024-11-28T02:54:44,059 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,060 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T02:54:44,060 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T02:54:44,061 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T02:54:44,064 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128056688b4535c43819379cee13e19d6cb_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128056688b4535c43819379cee13e19d6cb_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,064 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112837b6d36cebc64f32a000b826383dd759_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112837b6d36cebc64f32a000b826383dd759_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,065 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283daf31618ca04af089f8495a2b5902b9_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283daf31618ca04af089f8495a2b5902b9_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,066 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128444c6afe02104822b61dcb37b5b3cb34_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128444c6afe02104822b61dcb37b5b3cb34_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,067 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128456f1c36e76846e1be985263c7765512_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128456f1c36e76846e1be985263c7765512_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,068 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285414e4e1612a41a0bad02f396107c958_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285414e4e1612a41a0bad02f396107c958_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,069 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128802a114b13294307a23549f54f6f09e4_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128802a114b13294307a23549f54f6f09e4_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,070 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112884ecd5109e22426e949197129f79b5d3_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112884ecd5109e22426e949197129f79b5d3_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,071 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288ea784e996864d7fa76d44abdb3f9e3c_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288ea784e996864d7fa76d44abdb3f9e3c_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,072 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288ed74e4be70742738b7f556c7834aba0_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288ed74e4be70742738b7f556c7834aba0_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,073 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112891493b0d36204d0a8386e953846500d3_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112891493b0d36204d0a8386e953846500d3_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,073 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112893ffe98aa22a4a1c8b872e88ad6491f1_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112893ffe98aa22a4a1c8b872e88ad6491f1_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,074 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112894304269dded43c28870076093e5883d_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112894304269dded43c28870076093e5883d_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,075 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a7a7126b62f2433dbfd379649a8c3394_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a7a7126b62f2433dbfd379649a8c3394_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,076 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ae4d213238c645819d28ecaf7e251289_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ae4d213238c645819d28ecaf7e251289_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,077 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128df260e233b1b410483b974f1b3ec9178_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128df260e233b1b410483b974f1b3ec9178_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,078 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e5d8048db3c34783b2cb78729a78531c_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e5d8048db3c34783b2cb78729a78531c_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,079 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128edbcc3854cd54bbb96e9bdced74a3300_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128edbcc3854cd54bbb96e9bdced74a3300_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,080 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f05d778066d14fcaa2b906f9d83b602a_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f05d778066d14fcaa2b906f9d83b602a_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,081 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fa4ad5d662b44d5cab4eaaa2249f028a_f5c4bfa8bb8330a5297e82be419743be to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fa4ad5d662b44d5cab4eaaa2249f028a_f5c4bfa8bb8330a5297e82be419743be 2024-11-28T02:54:44,081 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T02:54:44,083 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:44,086 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T02:54:44,088 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T02:54:44,089 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:44,089 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T02:54:44,090 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732762484089"}]},"ts":"9223372036854775807"} 2024-11-28T02:54:44,093 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T02:54:44,093 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f5c4bfa8bb8330a5297e82be419743be, NAME => 'TestAcidGuarantees,,1732762450151.f5c4bfa8bb8330a5297e82be419743be.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T02:54:44,093 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T02:54:44,093 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732762484093"}]},"ts":"9223372036854775807"} 2024-11-28T02:54:44,095 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T02:54:44,096 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:44,098 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 65 msec 2024-11-28T02:54:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T02:54:44,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T02:54:44,149 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:44,156 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 249), OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=476 (was 521), ProcessCount=11 (was 11), AvailableMemoryMB=7450 (was 7683) 2024-11-28T02:54:44,165 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=476, ProcessCount=11, AvailableMemoryMB=7450 2024-11-28T02:54:44,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T02:54:44,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:54:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T02:54:44,169 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T02:54:44,169 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:44,169 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-28T02:54:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T02:54:44,170 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T02:54:44,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742043_1219 (size=1061) 2024-11-28T02:54:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T02:54:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T02:54:44,578 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381 2024-11-28T02:54:44,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742044_1220 (size=53) 2024-11-28T02:54:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T02:54:44,987 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:54:44,988 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 45a4746c6a5e848452a1dd3654f61765, disabling compactions & flushes 2024-11-28T02:54:44,988 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:44,988 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:44,988 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. after waiting 0 ms 2024-11-28T02:54:44,988 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:44,988 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:44,988 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 45a4746c6a5e848452a1dd3654f61765: Waiting for close lock at 1732762484987Disabling compacts and flushes for region at 1732762484987Disabling writes for close at 1732762484988 (+1 ms)Writing region close event to WAL at 1732762484988Closed at 1732762484988 2024-11-28T02:54:44,991 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T02:54:44,991 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732762484991"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732762484991"}]},"ts":"1732762484991"} 2024-11-28T02:54:44,995 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T02:54:44,998 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T02:54:44,998 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762484998"}]},"ts":"1732762484998"} 2024-11-28T02:54:45,000 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T02:54:45,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45a4746c6a5e848452a1dd3654f61765, ASSIGN}] 2024-11-28T02:54:45,002 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45a4746c6a5e848452a1dd3654f61765, ASSIGN 2024-11-28T02:54:45,002 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45a4746c6a5e848452a1dd3654f61765, ASSIGN; state=OFFLINE, location=978bd30859c3,41631,1732762419946; forceNewPlan=false, retain=false 2024-11-28T02:54:45,153 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=45a4746c6a5e848452a1dd3654f61765, regionState=OPENING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:54:45,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45a4746c6a5e848452a1dd3654f61765, ASSIGN because future has completed 2024-11-28T02:54:45,156 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:54:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T02:54:45,313 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:45,313 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:54:45,314 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:45,314 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:54:45,314 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:45,314 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:45,316 INFO [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:45,318 INFO [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:54:45,318 INFO [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45a4746c6a5e848452a1dd3654f61765 columnFamilyName A 2024-11-28T02:54:45,318 DEBUG [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:45,319 INFO [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] regionserver.HStore(327): Store=45a4746c6a5e848452a1dd3654f61765/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:54:45,319 INFO [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:45,320 INFO [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:54:45,320 INFO [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45a4746c6a5e848452a1dd3654f61765 columnFamilyName B 2024-11-28T02:54:45,320 DEBUG [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:45,321 INFO [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] regionserver.HStore(327): Store=45a4746c6a5e848452a1dd3654f61765/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:54:45,321 INFO [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:45,322 INFO [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:54:45,322 INFO [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45a4746c6a5e848452a1dd3654f61765 columnFamilyName C 2024-11-28T02:54:45,322 DEBUG [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:54:45,322 INFO [StoreOpener-45a4746c6a5e848452a1dd3654f61765-1 {}] regionserver.HStore(327): Store=45a4746c6a5e848452a1dd3654f61765/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:54:45,322 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:45,322 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:45,323 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:45,323 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:45,324 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:45,324 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:45,324 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T02:54:45,325 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:45,327 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:54:45,327 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 45a4746c6a5e848452a1dd3654f61765; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63061497, jitterRate=-0.06031046807765961}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T02:54:45,327 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:45,328 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 45a4746c6a5e848452a1dd3654f61765: Running coprocessor pre-open hook at 1732762485314Writing region info on filesystem at 1732762485314Initializing all the Stores at 1732762485315 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762485315Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762485316 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762485316Cleaning up temporary data from old regions at 1732762485324 (+8 ms)Running coprocessor post-open hooks at 1732762485327 (+3 ms)Region opened successfully at 1732762485328 (+1 ms) 2024-11-28T02:54:45,329 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., pid=63, masterSystemTime=1732762485308 2024-11-28T02:54:45,330 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:45,330 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:45,330 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=45a4746c6a5e848452a1dd3654f61765, regionState=OPEN, openSeqNum=2, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:54:45,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:54:45,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-28T02:54:45,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 in 177 msec 2024-11-28T02:54:45,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-28T02:54:45,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45a4746c6a5e848452a1dd3654f61765, ASSIGN in 333 msec 2024-11-28T02:54:45,336 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T02:54:45,336 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762485336"}]},"ts":"1732762485336"} 2024-11-28T02:54:45,338 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T02:54:45,338 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T02:54:45,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-11-28T02:54:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T02:54:46,316 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:46,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12616921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:46,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:46,318 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa53b40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:46,320 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16dcd39a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,321 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:46,323 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:46,324 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T02:54:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 978bd30859c3,32823,1732762419008 2024-11-28T02:54:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@652da493 2024-11-28T02:54:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T02:54:46,326 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T02:54:46,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c1be0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:46,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:46,332 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@157e1798, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:46,333 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70cbb255, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,334 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:46,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:46,335 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b2e17f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:46,336 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22f6a3aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,337 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:46,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:46,338 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176c5c2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:46,339 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1483e0f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:46,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:46,341 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ecb5bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:46,342 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e997ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:46,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:46,343 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fea6d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:46,345 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f51d00d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,346 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:46,346 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:46,347 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c754423, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:46,348 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1490f94c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:46,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:46,350 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@652e3aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:46,351 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48f51c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:46,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:46,352 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:46,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:46,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:46,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443edab5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:46,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:46,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:46,354 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a5f521, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:46,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:46,355 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:46,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435c2e27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:46,356 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:46,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e924c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,357 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:54:46,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:54:46,358 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:54:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:54:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:54:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7732c9a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:54:46,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:54:46,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:54:46,359 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:54:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3569bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:54:46,361 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:46,361 DEBUG [Thread-1052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:46,361 DEBUG [Thread-1047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:46,361 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:46,361 DEBUG [Thread-1060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:46,361 DEBUG [Thread-1043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:46,361 DEBUG [Thread-1041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:46,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:46,363 DEBUG [Thread-1049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:46,363 DEBUG [Thread-1045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:46,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:46,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:46,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:46,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:46,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:54:46,365 DEBUG [Thread-1045 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:46,366 DEBUG [Thread-1045 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:46,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:46,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:46,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:54:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:54:46,367 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:54:46,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-28T02:54:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:54:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:54:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:54:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T02:54:46,371 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:46,372 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:46,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:46,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:54:46,375 DEBUG [Thread-1056 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:46,376 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:54:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:54:46,377 DEBUG [Thread-1056 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:54:46,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:46,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:54:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:46,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:54:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:54:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:54:46,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:46,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:46,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:46,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:46,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:46,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:46,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/5c84fcf1416a403d9b5bc7d897f31982 is 50, key is test_row_0/A:col10/1732762486373/Put/seqid=0 2024-11-28T02:54:46,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1732762546407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762486403, max timestamp=1732762486414Num uniques 859; 2024-11-28T02:54:46,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762486403, max timestamp=1732762486414Num uniques 859; 2024-11-28T02:54:46,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:46,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:46,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762486403, max timestamp=1732762486414Num uniques 859; 2024-11-28T02:54:46,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:46,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762546439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1732762546439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762546439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742045_1221 (size=12001) 2024-11-28T02:54:46,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/5c84fcf1416a403d9b5bc7d897f31982 2024-11-28T02:54:46,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43874 deadline: 1732762546444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T02:54:46,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/d8bd4f2d212b42cc83781eda99d9c3a0 is 50, key is test_row_0/B:col10/1732762486373/Put/seqid=0 2024-11-28T02:54:46,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742046_1222 (size=12001) 2024-11-28T02:54:46,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/d8bd4f2d212b42cc83781eda99d9c3a0 2024-11-28T02:54:46,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T02:54:46,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:46,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:46,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:46,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:46,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:46,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1732762546526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/2f4e3d35935f4bf5b2a723ea9271ea5b is 50, key is test_row_0/C:col10/1732762486373/Put/seqid=0 2024-11-28T02:54:46,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742047_1223 (size=12001) 2024-11-28T02:54:46,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/2f4e3d35935f4bf5b2a723ea9271ea5b 2024-11-28T02:54:46,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/5c84fcf1416a403d9b5bc7d897f31982 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/5c84fcf1416a403d9b5bc7d897f31982 2024-11-28T02:54:46,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1732762546556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43874 deadline: 1732762546556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762546557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762546557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/5c84fcf1416a403d9b5bc7d897f31982, entries=150, sequenceid=13, filesize=11.7 K 2024-11-28T02:54:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/d8bd4f2d212b42cc83781eda99d9c3a0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d8bd4f2d212b42cc83781eda99d9c3a0 2024-11-28T02:54:46,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d8bd4f2d212b42cc83781eda99d9c3a0, entries=150, sequenceid=13, filesize=11.7 K 2024-11-28T02:54:46,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/2f4e3d35935f4bf5b2a723ea9271ea5b as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/2f4e3d35935f4bf5b2a723ea9271ea5b 2024-11-28T02:54:46,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/2f4e3d35935f4bf5b2a723ea9271ea5b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-28T02:54:46,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 45a4746c6a5e848452a1dd3654f61765 in 197ms, sequenceid=13, compaction requested=false 2024-11-28T02:54:46,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:46,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T02:54:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:46,680 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-11-28T02:54:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:46,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:46,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/a13e87e55d3e4ff79d22318e96c0f838 is 50, key is test_row_0/A:col10/1732762486436/Put/seqid=0 2024-11-28T02:54:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T02:54:46,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742048_1224 (size=12001) 2024-11-28T02:54:46,692 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/a13e87e55d3e4ff79d22318e96c0f838 2024-11-28T02:54:46,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/fda4358e5e0b4a7d84f9d5fc9bfbe5e2 is 50, key is test_row_0/B:col10/1732762486436/Put/seqid=0 2024-11-28T02:54:46,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742049_1225 (size=12001) 2024-11-28T02:54:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:46,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:46,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762546766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762546766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1732762546768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1732762546771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43874 deadline: 1732762546772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1732762546876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1732762546885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43874 deadline: 1732762546886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T02:54:47,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762547076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762547076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1732762547085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1732762547096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43874 deadline: 1732762547096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,105 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/fda4358e5e0b4a7d84f9d5fc9bfbe5e2 2024-11-28T02:54:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/9dce4bce31f9448e98f4d3ed2e1402f7 is 50, key is test_row_0/C:col10/1732762486436/Put/seqid=0 2024-11-28T02:54:47,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742050_1226 (size=12001) 2024-11-28T02:54:47,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1732762547396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1732762547405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43874 deadline: 1732762547405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T02:54:47,536 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/9dce4bce31f9448e98f4d3ed2e1402f7 2024-11-28T02:54:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/a13e87e55d3e4ff79d22318e96c0f838 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/a13e87e55d3e4ff79d22318e96c0f838 2024-11-28T02:54:47,547 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/a13e87e55d3e4ff79d22318e96c0f838, entries=150, sequenceid=48, filesize=11.7 K 2024-11-28T02:54:47,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/fda4358e5e0b4a7d84f9d5fc9bfbe5e2 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/fda4358e5e0b4a7d84f9d5fc9bfbe5e2 2024-11-28T02:54:47,552 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/fda4358e5e0b4a7d84f9d5fc9bfbe5e2, entries=150, sequenceid=48, filesize=11.7 K 2024-11-28T02:54:47,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/9dce4bce31f9448e98f4d3ed2e1402f7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/9dce4bce31f9448e98f4d3ed2e1402f7 2024-11-28T02:54:47,556 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/9dce4bce31f9448e98f4d3ed2e1402f7, entries=150, sequenceid=48, filesize=11.7 K 2024-11-28T02:54:47,557 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=67.09 KB/68700 for 45a4746c6a5e848452a1dd3654f61765 in 877ms, sequenceid=48, compaction requested=false 2024-11-28T02:54:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-28T02:54:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-28T02:54:47,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-28T02:54:47,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1870 sec 2024-11-28T02:54:47,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.1940 sec 2024-11-28T02:54:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:47,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T02:54:47,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:47,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:47,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:47,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:47,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:47,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:47,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/492d03859895401cbea23e3b301b76c8 is 50, key is test_row_0/A:col10/1732762486752/Put/seqid=0 2024-11-28T02:54:47,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742051_1227 (size=14341) 2024-11-28T02:54:47,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762547626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762547626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762547735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762547735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1732762547914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1732762547916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43874 deadline: 1732762547916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762547946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762547946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:48,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/492d03859895401cbea23e3b301b76c8 2024-11-28T02:54:48,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/6a4e1b5e4ef041378f35d1aa8d72fc7a is 50, key is test_row_0/B:col10/1732762486752/Put/seqid=0 2024-11-28T02:54:48,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742052_1228 (size=12001) 2024-11-28T02:54:48,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/6a4e1b5e4ef041378f35d1aa8d72fc7a 2024-11-28T02:54:48,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/8c6db4c7af464cf19480fdfea2bbbbda is 50, key is test_row_0/C:col10/1732762486752/Put/seqid=0 2024-11-28T02:54:48,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742053_1229 (size=12001) 2024-11-28T02:54:48,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762548256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762548256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:48,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/8c6db4c7af464cf19480fdfea2bbbbda 2024-11-28T02:54:48,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/492d03859895401cbea23e3b301b76c8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/492d03859895401cbea23e3b301b76c8 2024-11-28T02:54:48,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/492d03859895401cbea23e3b301b76c8, entries=200, sequenceid=63, filesize=14.0 K 2024-11-28T02:54:48,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/6a4e1b5e4ef041378f35d1aa8d72fc7a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6a4e1b5e4ef041378f35d1aa8d72fc7a 2024-11-28T02:54:48,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6a4e1b5e4ef041378f35d1aa8d72fc7a, entries=150, sequenceid=63, filesize=11.7 K 2024-11-28T02:54:48,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/8c6db4c7af464cf19480fdfea2bbbbda as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/8c6db4c7af464cf19480fdfea2bbbbda 2024-11-28T02:54:48,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/8c6db4c7af464cf19480fdfea2bbbbda, entries=150, sequenceid=63, filesize=11.7 K 2024-11-28T02:54:48,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 45a4746c6a5e848452a1dd3654f61765 in 860ms, sequenceid=63, compaction requested=true 2024-11-28T02:54:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:48,448 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:48,448 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:48,449 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:48,449 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:48,449 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/B is initiating minor compaction (all files) 2024-11-28T02:54:48,449 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/A is initiating minor compaction (all files) 2024-11-28T02:54:48,449 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/A in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:48,449 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/B in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:48,449 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/5c84fcf1416a403d9b5bc7d897f31982, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/a13e87e55d3e4ff79d22318e96c0f838, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/492d03859895401cbea23e3b301b76c8] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=37.4 K 2024-11-28T02:54:48,449 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d8bd4f2d212b42cc83781eda99d9c3a0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/fda4358e5e0b4a7d84f9d5fc9bfbe5e2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6a4e1b5e4ef041378f35d1aa8d72fc7a] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=35.2 K 2024-11-28T02:54:48,450 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c84fcf1416a403d9b5bc7d897f31982, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732762486373 2024-11-28T02:54:48,450 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting d8bd4f2d212b42cc83781eda99d9c3a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732762486373 2024-11-28T02:54:48,450 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a13e87e55d3e4ff79d22318e96c0f838, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732762486435 2024-11-28T02:54:48,450 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting fda4358e5e0b4a7d84f9d5fc9bfbe5e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732762486435 2024-11-28T02:54:48,450 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a4e1b5e4ef041378f35d1aa8d72fc7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732762486752 2024-11-28T02:54:48,450 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492d03859895401cbea23e3b301b76c8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732762486752 2024-11-28T02:54:48,457 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#A#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:48,458 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/4a6a99a7e750459fb15be6ee3754035c is 50, key is test_row_0/A:col10/1732762486752/Put/seqid=0 2024-11-28T02:54:48,459 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#B#compaction#187 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:48,460 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/c929117a7c764f4281578075410013ef is 50, key is test_row_0/B:col10/1732762486752/Put/seqid=0 2024-11-28T02:54:48,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742054_1230 (size=12104) 2024-11-28T02:54:48,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742055_1231 (size=12104) 2024-11-28T02:54:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T02:54:48,516 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:48,516 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-28T02:54:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T02:54:48,518 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:48,519 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:48,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T02:54:48,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-28T02:54:48,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:48,672 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T02:54:48,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:48,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:48,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:48,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:48,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:48,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:48,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/6557f22d6f47437c91c5378b076f1ec3 is 50, key is test_row_0/A:col10/1732762487624/Put/seqid=0 2024-11-28T02:54:48,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742056_1232 (size=12001) 2024-11-28T02:54:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:48,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:48,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762548818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:48,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762548819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T02:54:48,882 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/4a6a99a7e750459fb15be6ee3754035c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4a6a99a7e750459fb15be6ee3754035c 2024-11-28T02:54:48,890 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/c929117a7c764f4281578075410013ef as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/c929117a7c764f4281578075410013ef 2024-11-28T02:54:48,893 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/A of 45a4746c6a5e848452a1dd3654f61765 into 4a6a99a7e750459fb15be6ee3754035c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:48,893 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:48,893 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/A, priority=13, startTime=1732762488448; duration=0sec 2024-11-28T02:54:48,893 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:48,893 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:A 2024-11-28T02:54:48,893 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:48,895 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:48,895 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/C is initiating minor compaction (all files) 2024-11-28T02:54:48,895 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/C in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:48,895 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/2f4e3d35935f4bf5b2a723ea9271ea5b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/9dce4bce31f9448e98f4d3ed2e1402f7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/8c6db4c7af464cf19480fdfea2bbbbda] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=35.2 K 2024-11-28T02:54:48,896 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f4e3d35935f4bf5b2a723ea9271ea5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732762486373 2024-11-28T02:54:48,898 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dce4bce31f9448e98f4d3ed2e1402f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732762486435 2024-11-28T02:54:48,898 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c6db4c7af464cf19480fdfea2bbbbda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732762486752 2024-11-28T02:54:48,900 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/B of 45a4746c6a5e848452a1dd3654f61765 into c929117a7c764f4281578075410013ef(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:48,901 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:48,901 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/B, priority=13, startTime=1732762488448; duration=0sec 2024-11-28T02:54:48,901 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:48,901 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:B 2024-11-28T02:54:48,906 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#C#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:48,907 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/360fb0c4c38f4978bbd744efb634d51d is 50, key is test_row_0/C:col10/1732762486752/Put/seqid=0 2024-11-28T02:54:48,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742057_1233 (size=12104) 2024-11-28T02:54:48,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1732762548926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:48,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762548926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762548926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:48,929 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:48,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1732762548936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:48,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43874 deadline: 1732762548936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:48,940 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:48,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:48,940 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_1' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:48,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:49,082 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/6557f22d6f47437c91c5378b076f1ec3 2024-11-28T02:54:49,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/6252c83a43d4444d92192dc7832e677c is 50, key is test_row_0/B:col10/1732762487624/Put/seqid=0 2024-11-28T02:54:49,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742058_1234 (size=12001) 2024-11-28T02:54:49,101 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/6252c83a43d4444d92192dc7832e677c 2024-11-28T02:54:49,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/db9c974558a846f2bbc5e972963dcdc6 is 50, key is test_row_0/C:col10/1732762487624/Put/seqid=0 2024-11-28T02:54:49,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742059_1235 (size=12001) 2024-11-28T02:54:49,115 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/db9c974558a846f2bbc5e972963dcdc6 2024-11-28T02:54:49,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/6557f22d6f47437c91c5378b076f1ec3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/6557f22d6f47437c91c5378b076f1ec3 2024-11-28T02:54:49,135 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/6557f22d6f47437c91c5378b076f1ec3, entries=150, sequenceid=80, filesize=11.7 K 2024-11-28T02:54:49,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/6252c83a43d4444d92192dc7832e677c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6252c83a43d4444d92192dc7832e677c 2024-11-28T02:54:49,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:49,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762549136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:49,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762549136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:49,142 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6252c83a43d4444d92192dc7832e677c, entries=150, sequenceid=80, filesize=11.7 K 2024-11-28T02:54:49,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/db9c974558a846f2bbc5e972963dcdc6 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/db9c974558a846f2bbc5e972963dcdc6 2024-11-28T02:54:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T02:54:49,150 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/db9c974558a846f2bbc5e972963dcdc6, entries=150, sequenceid=80, filesize=11.7 K 2024-11-28T02:54:49,151 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 45a4746c6a5e848452a1dd3654f61765 in 479ms, sequenceid=80, compaction requested=false 2024-11-28T02:54:49,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:49,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:49,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-28T02:54:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-28T02:54:49,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-28T02:54:49,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 634 msec 2024-11-28T02:54:49,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 639 msec 2024-11-28T02:54:49,316 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/360fb0c4c38f4978bbd744efb634d51d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/360fb0c4c38f4978bbd744efb634d51d 2024-11-28T02:54:49,321 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/C of 45a4746c6a5e848452a1dd3654f61765 into 360fb0c4c38f4978bbd744efb634d51d(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:49,321 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:49,321 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/C, priority=13, startTime=1732762488448; duration=0sec 2024-11-28T02:54:49,321 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:49,321 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:C 2024-11-28T02:54:49,364 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T02:54:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:49,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T02:54:49,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:49,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:49,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:49,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:49,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:49,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:49,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/4978275cde2c4ab0a6c87a624b24b7ae is 50, key is test_row_0/A:col10/1732762489448/Put/seqid=0 2024-11-28T02:54:49,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742060_1236 (size=14341) 2024-11-28T02:54:49,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762549482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:49,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762549482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762549596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762549596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T02:54:49,656 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:49,656 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-28T02:54:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T02:54:49,658 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:49,659 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:49,659 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T02:54:49,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762549806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762549806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:49,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T02:54:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:49,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:49,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:49,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/4978275cde2c4ab0a6c87a624b24b7ae 2024-11-28T02:54:49,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/1f963928244d426bb1c8680a7ba1eb44 is 50, key is test_row_0/B:col10/1732762489448/Put/seqid=0 2024-11-28T02:54:49,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742061_1237 (size=12001) 2024-11-28T02:54:49,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T02:54:49,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:49,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:49,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:49,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:49,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T02:54:50,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T02:54:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:50,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:50,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762550115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762550116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:50,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/1f963928244d426bb1c8680a7ba1eb44 2024-11-28T02:54:50,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T02:54:50,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:50,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:50,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:50,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:50,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:50,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:50,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/079cf441d6cf494c96a988cdb3f1a3d9 is 50, key is test_row_0/C:col10/1732762489448/Put/seqid=0 2024-11-28T02:54:50,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742062_1238 (size=12001) 2024-11-28T02:54:50,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/079cf441d6cf494c96a988cdb3f1a3d9 2024-11-28T02:54:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T02:54:50,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/4978275cde2c4ab0a6c87a624b24b7ae as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4978275cde2c4ab0a6c87a624b24b7ae 2024-11-28T02:54:50,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4978275cde2c4ab0a6c87a624b24b7ae, entries=200, sequenceid=100, filesize=14.0 K 2024-11-28T02:54:50,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/1f963928244d426bb1c8680a7ba1eb44 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/1f963928244d426bb1c8680a7ba1eb44 2024-11-28T02:54:50,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/1f963928244d426bb1c8680a7ba1eb44, entries=150, sequenceid=100, filesize=11.7 K 2024-11-28T02:54:50,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/079cf441d6cf494c96a988cdb3f1a3d9 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/079cf441d6cf494c96a988cdb3f1a3d9 2024-11-28T02:54:50,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/079cf441d6cf494c96a988cdb3f1a3d9, entries=150, sequenceid=100, filesize=11.7 K 2024-11-28T02:54:50,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 45a4746c6a5e848452a1dd3654f61765 in 857ms, sequenceid=100, compaction requested=true 2024-11-28T02:54:50,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:50,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:50,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:50,307 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:50,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:50,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:50,307 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:50,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:50,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:50,309 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:50,309 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/B is initiating minor compaction (all files) 2024-11-28T02:54:50,310 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/B in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:50,310 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/c929117a7c764f4281578075410013ef, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6252c83a43d4444d92192dc7832e677c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/1f963928244d426bb1c8680a7ba1eb44] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=35.3 K 2024-11-28T02:54:50,310 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:50,310 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/A is initiating minor compaction (all files) 2024-11-28T02:54:50,310 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/A in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:50,310 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting c929117a7c764f4281578075410013ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732762486752 2024-11-28T02:54:50,310 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4a6a99a7e750459fb15be6ee3754035c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/6557f22d6f47437c91c5378b076f1ec3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4978275cde2c4ab0a6c87a624b24b7ae] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=37.5 K 2024-11-28T02:54:50,310 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6252c83a43d4444d92192dc7832e677c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732762487619 2024-11-28T02:54:50,310 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a6a99a7e750459fb15be6ee3754035c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732762486752 2024-11-28T02:54:50,311 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f963928244d426bb1c8680a7ba1eb44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732762488817 2024-11-28T02:54:50,311 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6557f22d6f47437c91c5378b076f1ec3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732762487619 2024-11-28T02:54:50,311 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4978275cde2c4ab0a6c87a624b24b7ae, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732762488817 2024-11-28T02:54:50,319 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#B#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:50,320 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/5cab72e4d0a2411d8ce9fe3deb256e56 is 50, key is test_row_0/B:col10/1732762489448/Put/seqid=0 2024-11-28T02:54:50,320 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#A#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:50,320 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/bfacfd9341494639ba67afbe757a3f22 is 50, key is test_row_0/A:col10/1732762489448/Put/seqid=0 2024-11-28T02:54:50,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742063_1239 (size=12207) 2024-11-28T02:54:50,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742064_1240 (size=12207) 2024-11-28T02:54:50,354 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/bfacfd9341494639ba67afbe757a3f22 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/bfacfd9341494639ba67afbe757a3f22 2024-11-28T02:54:50,358 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/A of 45a4746c6a5e848452a1dd3654f61765 into bfacfd9341494639ba67afbe757a3f22(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:50,358 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:50,358 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/A, priority=13, startTime=1732762490307; duration=0sec 2024-11-28T02:54:50,358 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:50,358 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:A 2024-11-28T02:54:50,359 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:50,360 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:50,360 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/C is initiating minor compaction (all files) 2024-11-28T02:54:50,360 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/C in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:50,360 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/360fb0c4c38f4978bbd744efb634d51d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/db9c974558a846f2bbc5e972963dcdc6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/079cf441d6cf494c96a988cdb3f1a3d9] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=35.3 K 2024-11-28T02:54:50,360 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 360fb0c4c38f4978bbd744efb634d51d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732762486752 2024-11-28T02:54:50,361 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting db9c974558a846f2bbc5e972963dcdc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732762487619 2024-11-28T02:54:50,361 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 079cf441d6cf494c96a988cdb3f1a3d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732762488817 2024-11-28T02:54:50,367 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#C#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:50,367 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/bffd413fb1754055a1f6123c6de07ff1 is 50, key is test_row_0/C:col10/1732762489448/Put/seqid=0 2024-11-28T02:54:50,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742065_1241 (size=12207) 2024-11-28T02:54:50,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T02:54:50,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:50,423 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T02:54:50,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:50,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:50,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:50,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:50,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:50,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:50,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/0763c00e878946ee9b1c62506afc81e8 is 50, key is test_row_0/A:col10/1732762489458/Put/seqid=0 2024-11-28T02:54:50,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742066_1242 (size=12001) 2024-11-28T02:54:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:50,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:50,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762550659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:50,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762550659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:50,740 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/5cab72e4d0a2411d8ce9fe3deb256e56 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/5cab72e4d0a2411d8ce9fe3deb256e56 2024-11-28T02:54:50,744 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/B of 45a4746c6a5e848452a1dd3654f61765 into 5cab72e4d0a2411d8ce9fe3deb256e56(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:50,744 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:50,744 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/B, priority=13, startTime=1732762490307; duration=0sec 2024-11-28T02:54:50,744 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:50,744 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:B 2024-11-28T02:54:50,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762550766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762550766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:50,784 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/bffd413fb1754055a1f6123c6de07ff1 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/bffd413fb1754055a1f6123c6de07ff1 2024-11-28T02:54:50,789 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/C of 45a4746c6a5e848452a1dd3654f61765 into bffd413fb1754055a1f6123c6de07ff1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:50,789 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:50,789 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/C, priority=13, startTime=1732762490307; duration=0sec 2024-11-28T02:54:50,789 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:50,789 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:C 2024-11-28T02:54:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T02:54:50,831 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/0763c00e878946ee9b1c62506afc81e8 2024-11-28T02:54:50,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/6ef2588c30d24e4fb15e87e242756466 is 50, key is test_row_0/B:col10/1732762489458/Put/seqid=0 2024-11-28T02:54:50,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742067_1243 (size=12001) 2024-11-28T02:54:50,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1732762550946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:50,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43874 deadline: 1732762550946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:50,949 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:50,949 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_1' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:50,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1732762550956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:50,957 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:50,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762550976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:50,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762550976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:50,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:51,243 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/6ef2588c30d24e4fb15e87e242756466 2024-11-28T02:54:51,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/489c91bb198b4031bef013b61cbed788 is 50, key is test_row_0/C:col10/1732762489458/Put/seqid=0 2024-11-28T02:54:51,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742068_1244 (size=12001) 2024-11-28T02:54:51,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762551286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:51,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:51,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:51,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:51,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762551286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:51,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:51,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:51,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:51,656 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/489c91bb198b4031bef013b61cbed788 2024-11-28T02:54:51,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/0763c00e878946ee9b1c62506afc81e8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/0763c00e878946ee9b1c62506afc81e8 2024-11-28T02:54:51,668 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/0763c00e878946ee9b1c62506afc81e8, entries=150, sequenceid=116, filesize=11.7 K 2024-11-28T02:54:51,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/6ef2588c30d24e4fb15e87e242756466 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6ef2588c30d24e4fb15e87e242756466 2024-11-28T02:54:51,674 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6ef2588c30d24e4fb15e87e242756466, entries=150, sequenceid=116, filesize=11.7 K 2024-11-28T02:54:51,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/489c91bb198b4031bef013b61cbed788 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/489c91bb198b4031bef013b61cbed788 2024-11-28T02:54:51,678 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/489c91bb198b4031bef013b61cbed788, entries=150, sequenceid=116, filesize=11.7 K 2024-11-28T02:54:51,679 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 45a4746c6a5e848452a1dd3654f61765 in 1256ms, sequenceid=116, compaction requested=false 2024-11-28T02:54:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-28T02:54:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-28T02:54:51,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-28T02:54:51,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0220 sec 2024-11-28T02:54:51,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.0280 sec 2024-11-28T02:54:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:51,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T02:54:51,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:51,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:51,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:51,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:51,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:51,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:51,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/2e5ee20f54e8473191fb1694c3b4f6c0 is 50, key is test_row_0/A:col10/1732762491798/Put/seqid=0 2024-11-28T02:54:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T02:54:51,806 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:51,807 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-28T02:54:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T02:54:51,809 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:51,810 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:51,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742069_1245 (size=14541) 2024-11-28T02:54:51,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:51,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762551821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:51,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:51,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/2e5ee20f54e8473191fb1694c3b4f6c0 2024-11-28T02:54:51,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762551823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:51,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:51,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:51,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:51,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/d76b33e54bd542a185201e4c5baad7b0 is 50, key is test_row_0/B:col10/1732762491798/Put/seqid=0 2024-11-28T02:54:51,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742070_1246 (size=12151) 2024-11-28T02:54:51,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/d76b33e54bd542a185201e4c5baad7b0 2024-11-28T02:54:51,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/10254d6541f74271b1f8a4961599da86 is 50, key is test_row_0/C:col10/1732762491798/Put/seqid=0 2024-11-28T02:54:51,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742071_1247 (size=12151) 2024-11-28T02:54:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T02:54:51,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762551926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:51,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762551936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:51,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T02:54:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:51,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:52,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T02:54:52,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:52,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:52,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:52,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:52,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T02:54:52,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762552135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:52,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:52,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762552146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:52,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:52,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/10254d6541f74271b1f8a4961599da86 2024-11-28T02:54:52,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/2e5ee20f54e8473191fb1694c3b4f6c0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/2e5ee20f54e8473191fb1694c3b4f6c0 2024-11-28T02:54:52,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/2e5ee20f54e8473191fb1694c3b4f6c0, entries=200, sequenceid=137, filesize=14.2 K 2024-11-28T02:54:52,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T02:54:52,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:52,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:52,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:52,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:52,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:52,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/d76b33e54bd542a185201e4c5baad7b0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d76b33e54bd542a185201e4c5baad7b0 2024-11-28T02:54:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:52,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d76b33e54bd542a185201e4c5baad7b0, entries=150, sequenceid=137, filesize=11.9 K 2024-11-28T02:54:52,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/10254d6541f74271b1f8a4961599da86 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/10254d6541f74271b1f8a4961599da86 2024-11-28T02:54:52,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/10254d6541f74271b1f8a4961599da86, entries=150, sequenceid=137, filesize=11.9 K 2024-11-28T02:54:52,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 45a4746c6a5e848452a1dd3654f61765 in 481ms, sequenceid=137, compaction requested=true 2024-11-28T02:54:52,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:52,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:52,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:52,279 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:52,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:52,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:52,279 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:52,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:52,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:52,281 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:52,281 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:52,281 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/B is initiating minor compaction (all files) 2024-11-28T02:54:52,281 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/A is initiating minor compaction (all files) 2024-11-28T02:54:52,281 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/B in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:52,281 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/A in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:52,281 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/5cab72e4d0a2411d8ce9fe3deb256e56, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6ef2588c30d24e4fb15e87e242756466, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d76b33e54bd542a185201e4c5baad7b0] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=35.5 K 2024-11-28T02:54:52,281 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/bfacfd9341494639ba67afbe757a3f22, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/0763c00e878946ee9b1c62506afc81e8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/2e5ee20f54e8473191fb1694c3b4f6c0] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=37.8 K 2024-11-28T02:54:52,282 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfacfd9341494639ba67afbe757a3f22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732762488817 2024-11-28T02:54:52,282 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cab72e4d0a2411d8ce9fe3deb256e56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732762488817 2024-11-28T02:54:52,282 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef2588c30d24e4fb15e87e242756466, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732762489458 2024-11-28T02:54:52,282 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0763c00e878946ee9b1c62506afc81e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732762489458 2024-11-28T02:54:52,282 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting d76b33e54bd542a185201e4c5baad7b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732762490658 2024-11-28T02:54:52,282 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e5ee20f54e8473191fb1694c3b4f6c0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732762490652 2024-11-28T02:54:52,289 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:52,290 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/5d04e689971443c7b6521dc7fda9091a is 50, key is test_row_0/B:col10/1732762491798/Put/seqid=0 2024-11-28T02:54:52,293 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#A#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:52,294 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/ec92e339a88844f9937282cf5297e46f is 50, key is test_row_0/A:col10/1732762491798/Put/seqid=0 2024-11-28T02:54:52,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742072_1248 (size=12459) 2024-11-28T02:54:52,304 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/5d04e689971443c7b6521dc7fda9091a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/5d04e689971443c7b6521dc7fda9091a 2024-11-28T02:54:52,312 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/B of 45a4746c6a5e848452a1dd3654f61765 into 5d04e689971443c7b6521dc7fda9091a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:52,312 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:52,312 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/B, priority=13, startTime=1732762492279; duration=0sec 2024-11-28T02:54:52,312 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:52,312 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:B 2024-11-28T02:54:52,312 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:52,314 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:52,314 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/C is initiating minor compaction (all files) 2024-11-28T02:54:52,314 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/C in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:52,314 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/bffd413fb1754055a1f6123c6de07ff1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/489c91bb198b4031bef013b61cbed788, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/10254d6541f74271b1f8a4961599da86] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=35.5 K 2024-11-28T02:54:52,315 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting bffd413fb1754055a1f6123c6de07ff1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732762488817 2024-11-28T02:54:52,315 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 489c91bb198b4031bef013b61cbed788, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732762489458 2024-11-28T02:54:52,316 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 10254d6541f74271b1f8a4961599da86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732762490658 2024-11-28T02:54:52,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742073_1249 (size=12459) 2024-11-28T02:54:52,326 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#C#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:52,326 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/3c02923568964ecca263f46037061d18 is 50, key is test_row_0/C:col10/1732762491798/Put/seqid=0 2024-11-28T02:54:52,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742074_1250 (size=12459) 2024-11-28T02:54:52,342 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/3c02923568964ecca263f46037061d18 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3c02923568964ecca263f46037061d18 2024-11-28T02:54:52,348 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/C of 45a4746c6a5e848452a1dd3654f61765 into 3c02923568964ecca263f46037061d18(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:52,348 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:52,348 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/C, priority=13, startTime=1732762492279; duration=0sec 2024-11-28T02:54:52,348 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:52,348 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:C 2024-11-28T02:54:52,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T02:54:52,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:52,421 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T02:54:52,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:52,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:52,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:52,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:52,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:52,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:52,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/eafa880ee55c4a1c966f0a31adb6e91a is 50, key is test_row_0/A:col10/1732762491821/Put/seqid=0 2024-11-28T02:54:52,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742075_1251 (size=12151) 2024-11-28T02:54:52,433 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/eafa880ee55c4a1c966f0a31adb6e91a 2024-11-28T02:54:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T02:54:52,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/a0b80cbdf3984e1aa68c80ab6a4a665a is 50, key is test_row_0/B:col10/1732762491821/Put/seqid=0 2024-11-28T02:54:52,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742076_1252 (size=12151) 2024-11-28T02:54:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:52,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:52,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762552488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:52,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762552492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:52,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:52,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762552596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:52,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762552606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:52,726 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/ec92e339a88844f9937282cf5297e46f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/ec92e339a88844f9937282cf5297e46f 2024-11-28T02:54:52,730 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/A of 45a4746c6a5e848452a1dd3654f61765 into ec92e339a88844f9937282cf5297e46f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:52,730 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:52,730 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/A, priority=13, startTime=1732762492279; duration=0sec 2024-11-28T02:54:52,731 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:52,731 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:A 2024-11-28T02:54:52,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762552806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:52,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762552816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:52,846 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/a0b80cbdf3984e1aa68c80ab6a4a665a 2024-11-28T02:54:52,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/2f027c2368e548d38641d7f0050459c2 is 50, key is test_row_0/C:col10/1732762491821/Put/seqid=0 2024-11-28T02:54:52,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742077_1253 (size=12151) 2024-11-28T02:54:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T02:54:53,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:53,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762553115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:53,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762553126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:53,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:53,270 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/2f027c2368e548d38641d7f0050459c2 2024-11-28T02:54:53,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/eafa880ee55c4a1c966f0a31adb6e91a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/eafa880ee55c4a1c966f0a31adb6e91a 2024-11-28T02:54:53,278 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/eafa880ee55c4a1c966f0a31adb6e91a, entries=150, sequenceid=152, filesize=11.9 K 2024-11-28T02:54:53,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/a0b80cbdf3984e1aa68c80ab6a4a665a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/a0b80cbdf3984e1aa68c80ab6a4a665a 2024-11-28T02:54:53,282 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/a0b80cbdf3984e1aa68c80ab6a4a665a, entries=150, sequenceid=152, filesize=11.9 K 2024-11-28T02:54:53,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/2f027c2368e548d38641d7f0050459c2 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/2f027c2368e548d38641d7f0050459c2 2024-11-28T02:54:53,287 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/2f027c2368e548d38641d7f0050459c2, entries=150, sequenceid=152, filesize=11.9 K 2024-11-28T02:54:53,287 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 45a4746c6a5e848452a1dd3654f61765 in 866ms, sequenceid=152, compaction requested=false 2024-11-28T02:54:53,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:53,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:53,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-28T02:54:53,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-28T02:54:53,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-28T02:54:53,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4780 sec 2024-11-28T02:54:53,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4830 sec 2024-11-28T02:54:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:53,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T02:54:53,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:53,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:53,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:53,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:53,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:53,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:53,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/91639a1741ae4f4d9bc5f51398ee9b36 is 50, key is test_row_0/A:col10/1732762492487/Put/seqid=0 2024-11-28T02:54:53,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742078_1254 (size=12151) 2024-11-28T02:54:53,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762553648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:53,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762553650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:53,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762553756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:53,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762553766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T02:54:53,966 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:53,966 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-28T02:54:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T02:54:53,968 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:53,969 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:53,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:53,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762553966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:53,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:53,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762553976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:53,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:53,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:54,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/91639a1741ae4f4d9bc5f51398ee9b36 2024-11-28T02:54:54,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/68ebe7f6aa24485885ee4802429e0ab1 is 50, key is test_row_0/B:col10/1732762492487/Put/seqid=0 2024-11-28T02:54:54,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742079_1255 (size=12151) 2024-11-28T02:54:54,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/68ebe7f6aa24485885ee4802429e0ab1 2024-11-28T02:54:54,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/ab6d6709a9ce47a3903efb5a49a0d4ec is 50, key is test_row_0/C:col10/1732762492487/Put/seqid=0 2024-11-28T02:54:54,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742080_1256 (size=12151) 2024-11-28T02:54:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T02:54:54,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T02:54:54,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:54,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:54,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:54,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:54,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:54,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T02:54:54,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:54,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:54,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:54,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:54,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:54,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762554275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T02:54:54,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762554286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:54,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T02:54:54,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:54,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:54,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:54,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:54,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:54,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/ab6d6709a9ce47a3903efb5a49a0d4ec 2024-11-28T02:54:54,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/91639a1741ae4f4d9bc5f51398ee9b36 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/91639a1741ae4f4d9bc5f51398ee9b36 2024-11-28T02:54:54,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/91639a1741ae4f4d9bc5f51398ee9b36, entries=150, sequenceid=173, filesize=11.9 K 2024-11-28T02:54:54,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/68ebe7f6aa24485885ee4802429e0ab1 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/68ebe7f6aa24485885ee4802429e0ab1 2024-11-28T02:54:54,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/68ebe7f6aa24485885ee4802429e0ab1, entries=150, sequenceid=173, filesize=11.9 K 2024-11-28T02:54:54,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/ab6d6709a9ce47a3903efb5a49a0d4ec as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/ab6d6709a9ce47a3903efb5a49a0d4ec 2024-11-28T02:54:54,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/ab6d6709a9ce47a3903efb5a49a0d4ec, entries=150, sequenceid=173, filesize=11.9 K 2024-11-28T02:54:54,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 45a4746c6a5e848452a1dd3654f61765 in 854ms, sequenceid=173, compaction requested=true 2024-11-28T02:54:54,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:54,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:54,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:54,483 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:54,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:54,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:54,483 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:54,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:54,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:54,484 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:54,484 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:54,484 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/B is initiating minor compaction (all files) 2024-11-28T02:54:54,484 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/A is initiating minor compaction (all files) 2024-11-28T02:54:54,484 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/A in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:54,484 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/B in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:54,484 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/ec92e339a88844f9937282cf5297e46f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/eafa880ee55c4a1c966f0a31adb6e91a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/91639a1741ae4f4d9bc5f51398ee9b36] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=35.9 K 2024-11-28T02:54:54,484 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/5d04e689971443c7b6521dc7fda9091a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/a0b80cbdf3984e1aa68c80ab6a4a665a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/68ebe7f6aa24485885ee4802429e0ab1] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=35.9 K 2024-11-28T02:54:54,485 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d04e689971443c7b6521dc7fda9091a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732762490658 2024-11-28T02:54:54,485 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec92e339a88844f9937282cf5297e46f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732762490658 2024-11-28T02:54:54,485 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting a0b80cbdf3984e1aa68c80ab6a4a665a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732762491818 2024-11-28T02:54:54,485 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting eafa880ee55c4a1c966f0a31adb6e91a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732762491818 2024-11-28T02:54:54,485 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 68ebe7f6aa24485885ee4802429e0ab1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732762492487 2024-11-28T02:54:54,485 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91639a1741ae4f4d9bc5f51398ee9b36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732762492487 2024-11-28T02:54:54,496 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#A#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:54,497 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:54,497 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/de035d84b36449b499bae80a4eb86578 is 50, key is test_row_0/A:col10/1732762492487/Put/seqid=0 2024-11-28T02:54:54,497 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/8f50c3f9667944449d2f3b09ad94b3e4 is 50, key is test_row_0/B:col10/1732762492487/Put/seqid=0 2024-11-28T02:54:54,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742081_1257 (size=12561) 2024-11-28T02:54:54,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742082_1258 (size=12561) 2024-11-28T02:54:54,536 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/de035d84b36449b499bae80a4eb86578 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/de035d84b36449b499bae80a4eb86578 2024-11-28T02:54:54,537 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/8f50c3f9667944449d2f3b09ad94b3e4 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8f50c3f9667944449d2f3b09ad94b3e4 2024-11-28T02:54:54,543 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/A of 45a4746c6a5e848452a1dd3654f61765 into de035d84b36449b499bae80a4eb86578(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:54,543 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:54,543 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/A, priority=13, startTime=1732762494483; duration=0sec 2024-11-28T02:54:54,543 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/B of 45a4746c6a5e848452a1dd3654f61765 into 8f50c3f9667944449d2f3b09ad94b3e4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:54,543 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:54,543 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/B, priority=13, startTime=1732762494483; duration=0sec 2024-11-28T02:54:54,543 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:54,543 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:A 2024-11-28T02:54:54,543 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:54,543 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:54,543 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:B 2024-11-28T02:54:54,545 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:54,545 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/C is initiating minor compaction (all files) 2024-11-28T02:54:54,545 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/C in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:54,545 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3c02923568964ecca263f46037061d18, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/2f027c2368e548d38641d7f0050459c2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/ab6d6709a9ce47a3903efb5a49a0d4ec] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=35.9 K 2024-11-28T02:54:54,545 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c02923568964ecca263f46037061d18, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732762490658 2024-11-28T02:54:54,546 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f027c2368e548d38641d7f0050459c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732762491818 2024-11-28T02:54:54,546 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab6d6709a9ce47a3903efb5a49a0d4ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732762492487 2024-11-28T02:54:54,556 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#C#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:54,556 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/d81d0a92b59841b7ab2a3c2e3149c7e6 is 50, key is test_row_0/C:col10/1732762492487/Put/seqid=0 2024-11-28T02:54:54,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742083_1259 (size=12561) 2024-11-28T02:54:54,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T02:54:54,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:54,581 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T02:54:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:54,581 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/d81d0a92b59841b7ab2a3c2e3149c7e6 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/d81d0a92b59841b7ab2a3c2e3149c7e6 2024-11-28T02:54:54,591 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/C of 45a4746c6a5e848452a1dd3654f61765 into d81d0a92b59841b7ab2a3c2e3149c7e6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:54,591 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:54,591 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/C, priority=13, startTime=1732762494483; duration=0sec 2024-11-28T02:54:54,591 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:54,591 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:C 2024-11-28T02:54:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T02:54:54,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/1c04af4536d84620897af5e72f3bbb0a is 50, key is test_row_1/A:col10/1732762493647/Put/seqid=0 2024-11-28T02:54:54,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742084_1260 (size=9757) 2024-11-28T02:54:54,602 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/1c04af4536d84620897af5e72f3bbb0a 2024-11-28T02:54:54,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/4d8da61c03554b2ba203990caf739ad8 is 50, key is test_row_1/B:col10/1732762493647/Put/seqid=0 2024-11-28T02:54:54,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742085_1261 (size=9757) 2024-11-28T02:54:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:54,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:54,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762554813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762554815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:54,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762554926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762554926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:54,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1732762554966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:54,969 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:54,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43874 deadline: 1732762554986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:54,990 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_1' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:54,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1732762554996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:54,997 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:55,016 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/4d8da61c03554b2ba203990caf739ad8 2024-11-28T02:54:55,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/ce78c72f93c543f8a5b5438bf31b730e is 50, key is test_row_1/C:col10/1732762493647/Put/seqid=0 2024-11-28T02:54:55,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742086_1262 (size=9757) 2024-11-28T02:54:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T02:54:55,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762555135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:55,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762555136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:55,438 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/ce78c72f93c543f8a5b5438bf31b730e 2024-11-28T02:54:55,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/1c04af4536d84620897af5e72f3bbb0a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/1c04af4536d84620897af5e72f3bbb0a 2024-11-28T02:54:55,445 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/1c04af4536d84620897af5e72f3bbb0a, entries=100, sequenceid=187, filesize=9.5 K 2024-11-28T02:54:55,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762555446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:55,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:55,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:55,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762555446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:55,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:55,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:55,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:55,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:55,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/4d8da61c03554b2ba203990caf739ad8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/4d8da61c03554b2ba203990caf739ad8 2024-11-28T02:54:55,453 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/4d8da61c03554b2ba203990caf739ad8, entries=100, sequenceid=187, filesize=9.5 K 2024-11-28T02:54:55,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/ce78c72f93c543f8a5b5438bf31b730e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/ce78c72f93c543f8a5b5438bf31b730e 2024-11-28T02:54:55,457 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/ce78c72f93c543f8a5b5438bf31b730e, entries=100, sequenceid=187, filesize=9.5 K 2024-11-28T02:54:55,458 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 45a4746c6a5e848452a1dd3654f61765 in 878ms, sequenceid=187, compaction requested=false 2024-11-28T02:54:55,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:55,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:55,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-28T02:54:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-28T02:54:55,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-28T02:54:55,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4910 sec 2024-11-28T02:54:55,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4950 sec 2024-11-28T02:54:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:55,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T02:54:55,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:55,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:55,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:55,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:55,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:55,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:55,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/f8002d7059db45d5aace1d22af12935a is 50, key is test_row_0/A:col10/1732762494814/Put/seqid=0 2024-11-28T02:54:55,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742087_1263 (size=12151) 2024-11-28T02:54:55,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762555976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:55,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762555977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:55,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:55,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:55,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:56,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:56,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762556085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762556086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T02:54:56,116 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:56,117 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-28T02:54:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T02:54:56,118 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:56,119 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:56,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T02:54:56,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T02:54:56,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:56,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:56,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:56,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:56,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:54:56,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762556295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762556296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:56,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/f8002d7059db45d5aace1d22af12935a 2024-11-28T02:54:56,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/f1ba7e41e10b4b708a3de86a42e1c678 is 50, key is test_row_0/B:col10/1732762494814/Put/seqid=0 2024-11-28T02:54:56,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742088_1264 (size=12151) 2024-11-28T02:54:56,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/f1ba7e41e10b4b708a3de86a42e1c678 2024-11-28T02:54:56,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/78f671a23d8040ea8466686d00d00b81 is 50, key is test_row_0/C:col10/1732762494814/Put/seqid=0 2024-11-28T02:54:56,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742089_1265 (size=12151) 2024-11-28T02:54:56,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/78f671a23d8040ea8466686d00d00b81 2024-11-28T02:54:56,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/f8002d7059db45d5aace1d22af12935a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/f8002d7059db45d5aace1d22af12935a 2024-11-28T02:54:56,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/f8002d7059db45d5aace1d22af12935a, entries=150, sequenceid=209, filesize=11.9 K 2024-11-28T02:54:56,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/f1ba7e41e10b4b708a3de86a42e1c678 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/f1ba7e41e10b4b708a3de86a42e1c678 2024-11-28T02:54:56,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/f1ba7e41e10b4b708a3de86a42e1c678, entries=150, sequenceid=209, filesize=11.9 K 2024-11-28T02:54:56,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/78f671a23d8040ea8466686d00d00b81 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/78f671a23d8040ea8466686d00d00b81 2024-11-28T02:54:56,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/78f671a23d8040ea8466686d00d00b81, entries=150, sequenceid=209, filesize=11.9 K 2024-11-28T02:54:56,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 45a4746c6a5e848452a1dd3654f61765 in 461ms, sequenceid=209, compaction requested=true 2024-11-28T02:54:56,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:56,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:56,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:56,420 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:56,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:56,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:56,421 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:56,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:56,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:56,423 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:56,423 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/B is initiating minor compaction (all files) 2024-11-28T02:54:56,423 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/B in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:56,423 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:56,423 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/A is initiating minor compaction (all files) 2024-11-28T02:54:56,423 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8f50c3f9667944449d2f3b09ad94b3e4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/4d8da61c03554b2ba203990caf739ad8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/f1ba7e41e10b4b708a3de86a42e1c678] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=33.7 K 2024-11-28T02:54:56,423 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/A in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:56,423 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/de035d84b36449b499bae80a4eb86578, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/1c04af4536d84620897af5e72f3bbb0a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/f8002d7059db45d5aace1d22af12935a] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=33.7 K 2024-11-28T02:54:56,424 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f50c3f9667944449d2f3b09ad94b3e4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732762492487 2024-11-28T02:54:56,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T02:54:56,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:56,424 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d8da61c03554b2ba203990caf739ad8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732762493647 2024-11-28T02:54:56,424 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:54:56,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:56,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:56,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:56,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:56,425 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting de035d84b36449b499bae80a4eb86578, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732762492487 2024-11-28T02:54:56,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:56,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:56,425 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c04af4536d84620897af5e72f3bbb0a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732762493647 2024-11-28T02:54:56,425 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ba7e41e10b4b708a3de86a42e1c678, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732762494812 2024-11-28T02:54:56,426 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8002d7059db45d5aace1d22af12935a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732762494812 2024-11-28T02:54:56,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/8cb7be5fc7064cf5befc7795f0155f44 is 50, key is test_row_0/A:col10/1732762495962/Put/seqid=0 2024-11-28T02:54:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T02:54:56,438 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#A#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:56,438 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#B#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:56,439 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/95c144c4fe0d4e49b75fbb0a30be1e28 is 50, key is test_row_0/A:col10/1732762494814/Put/seqid=0 2024-11-28T02:54:56,439 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/d7ea00d481304eb4a40bf7d8ff3027e2 is 50, key is test_row_0/B:col10/1732762494814/Put/seqid=0 2024-11-28T02:54:56,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742092_1268 (size=12663) 2024-11-28T02:54:56,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742091_1267 (size=12663) 2024-11-28T02:54:56,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742090_1266 (size=12151) 2024-11-28T02:54:56,450 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/d7ea00d481304eb4a40bf7d8ff3027e2 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d7ea00d481304eb4a40bf7d8ff3027e2 2024-11-28T02:54:56,453 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/95c144c4fe0d4e49b75fbb0a30be1e28 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/95c144c4fe0d4e49b75fbb0a30be1e28 2024-11-28T02:54:56,457 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/B of 45a4746c6a5e848452a1dd3654f61765 into d7ea00d481304eb4a40bf7d8ff3027e2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:56,457 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/A of 45a4746c6a5e848452a1dd3654f61765 into 95c144c4fe0d4e49b75fbb0a30be1e28(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:56,457 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:56,457 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:56,457 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/B, priority=13, startTime=1732762496420; duration=0sec 2024-11-28T02:54:56,457 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/A, priority=13, startTime=1732762496420; duration=0sec 2024-11-28T02:54:56,457 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:56,457 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:B 2024-11-28T02:54:56,457 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:56,457 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:56,458 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:A 2024-11-28T02:54:56,459 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:56,459 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/C is initiating minor compaction (all files) 2024-11-28T02:54:56,459 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/C in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:56,459 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/d81d0a92b59841b7ab2a3c2e3149c7e6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/ce78c72f93c543f8a5b5438bf31b730e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/78f671a23d8040ea8466686d00d00b81] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=33.7 K 2024-11-28T02:54:56,459 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting d81d0a92b59841b7ab2a3c2e3149c7e6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732762492487 2024-11-28T02:54:56,460 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting ce78c72f93c543f8a5b5438bf31b730e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732762493647 2024-11-28T02:54:56,460 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 78f671a23d8040ea8466686d00d00b81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732762494812 2024-11-28T02:54:56,472 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#C#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:56,472 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/af044a4ed42e4a0c8e56ad3ea0369a9b is 50, key is test_row_0/C:col10/1732762494814/Put/seqid=0 2024-11-28T02:54:56,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742093_1269 (size=12663) 2024-11-28T02:54:56,485 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/af044a4ed42e4a0c8e56ad3ea0369a9b as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/af044a4ed42e4a0c8e56ad3ea0369a9b 2024-11-28T02:54:56,490 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/C of 45a4746c6a5e848452a1dd3654f61765 into af044a4ed42e4a0c8e56ad3ea0369a9b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:56,490 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:56,490 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/C, priority=13, startTime=1732762496421; duration=0sec 2024-11-28T02:54:56,490 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:56,490 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:C 2024-11-28T02:54:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:56,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:56,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762556645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:56,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762496625, max timestamp=1732762496648Num uniques 859; 2024-11-28T02:54:56,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:54:56,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762496625, max timestamp=1732762496648Num uniques 859; 2024-11-28T02:54:56,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762496625, max timestamp=1732762496648Num uniques 859; 2024-11-28T02:54:56,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:56,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:54:56,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762556728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:56,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T02:54:56,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762556756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:56,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:56,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762556836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:56,849 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/8cb7be5fc7064cf5befc7795f0155f44 2024-11-28T02:54:56,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/712110eb485b48bda2d1fcb6f1a515b7 is 50, key is test_row_0/B:col10/1732762495962/Put/seqid=0 2024-11-28T02:54:56,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742094_1270 (size=12151) 2024-11-28T02:54:56,858 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/712110eb485b48bda2d1fcb6f1a515b7 2024-11-28T02:54:56,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/941416e5e2af45cda69112234e4495fa is 50, key is test_row_0/C:col10/1732762495962/Put/seqid=0 2024-11-28T02:54:56,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742095_1271 (size=12151) 2024-11-28T02:54:56,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762556966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:57,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762557046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:57,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:57,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:57,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T02:54:57,268 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/941416e5e2af45cda69112234e4495fa 2024-11-28T02:54:57,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/8cb7be5fc7064cf5befc7795f0155f44 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/8cb7be5fc7064cf5befc7795f0155f44 2024-11-28T02:54:57,275 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/8cb7be5fc7064cf5befc7795f0155f44, entries=150, sequenceid=220, filesize=11.9 K 2024-11-28T02:54:57,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/712110eb485b48bda2d1fcb6f1a515b7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/712110eb485b48bda2d1fcb6f1a515b7 2024-11-28T02:54:57,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762557275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:57,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:57,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:57,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:57,280 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/712110eb485b48bda2d1fcb6f1a515b7, entries=150, sequenceid=220, filesize=11.9 K 2024-11-28T02:54:57,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/941416e5e2af45cda69112234e4495fa as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/941416e5e2af45cda69112234e4495fa 2024-11-28T02:54:57,285 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/941416e5e2af45cda69112234e4495fa, entries=150, sequenceid=220, filesize=11.9 K 2024-11-28T02:54:57,286 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 45a4746c6a5e848452a1dd3654f61765 in 862ms, sequenceid=220, compaction requested=false 2024-11-28T02:54:57,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:57,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:57,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-28T02:54:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-28T02:54:57,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-28T02:54:57,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1680 sec 2024-11-28T02:54:57,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.1720 sec 2024-11-28T02:54:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:57,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-11-28T02:54:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:57,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:54:57,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/18506f84b2804dfdb4a38f7cc103769f is 50, key is test_row_0/A:col10/1732762497358/Put/seqid=0 2024-11-28T02:54:57,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742096_1272 (size=14641) 2024-11-28T02:54:57,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762557393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:57,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:57,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:57,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:57,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762557506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:57,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:57,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:57,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:57,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762557716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:57,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/18506f84b2804dfdb4a38f7cc103769f 2024-11-28T02:54:57,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/1af2f4f50ff945aab3bc253a91f4cf20 is 50, key is test_row_0/B:col10/1732762497358/Put/seqid=0 2024-11-28T02:54:57,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742097_1273 (size=12201) 2024-11-28T02:54:57,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762557786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:58,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:58,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762558025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:58,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/1af2f4f50ff945aab3bc253a91f4cf20 2024-11-28T02:54:58,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/932ce8bf0417475194ba7df919bda35a is 50, key is test_row_0/C:col10/1732762497358/Put/seqid=0 2024-11-28T02:54:58,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742098_1274 (size=12201) 2024-11-28T02:54:58,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/932ce8bf0417475194ba7df919bda35a 2024-11-28T02:54:58,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/18506f84b2804dfdb4a38f7cc103769f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/18506f84b2804dfdb4a38f7cc103769f 2024-11-28T02:54:58,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/18506f84b2804dfdb4a38f7cc103769f, entries=200, sequenceid=259, filesize=14.3 K 2024-11-28T02:54:58,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/1af2f4f50ff945aab3bc253a91f4cf20 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/1af2f4f50ff945aab3bc253a91f4cf20 2024-11-28T02:54:58,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/1af2f4f50ff945aab3bc253a91f4cf20, entries=150, sequenceid=259, filesize=11.9 K 2024-11-28T02:54:58,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/932ce8bf0417475194ba7df919bda35a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/932ce8bf0417475194ba7df919bda35a 2024-11-28T02:54:58,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/932ce8bf0417475194ba7df919bda35a, entries=150, sequenceid=259, filesize=11.9 K 2024-11-28T02:54:58,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for 45a4746c6a5e848452a1dd3654f61765 in 851ms, sequenceid=259, compaction requested=true 2024-11-28T02:54:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:54:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:58,210 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:54:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:58,210 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:54:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:58,211 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:58,211 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/B is initiating minor compaction (all files) 2024-11-28T02:54:58,211 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/B in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:58,212 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d7ea00d481304eb4a40bf7d8ff3027e2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/712110eb485b48bda2d1fcb6f1a515b7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/1af2f4f50ff945aab3bc253a91f4cf20] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=36.1 K 2024-11-28T02:54:58,212 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting d7ea00d481304eb4a40bf7d8ff3027e2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732762494812 2024-11-28T02:54:58,212 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:58,212 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/A is initiating minor compaction (all files) 2024-11-28T02:54:58,212 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/A in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:58,212 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 712110eb485b48bda2d1fcb6f1a515b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732762495962 2024-11-28T02:54:58,212 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/95c144c4fe0d4e49b75fbb0a30be1e28, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/8cb7be5fc7064cf5befc7795f0155f44, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/18506f84b2804dfdb4a38f7cc103769f] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=38.5 K 2024-11-28T02:54:58,212 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95c144c4fe0d4e49b75fbb0a30be1e28, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732762494812 2024-11-28T02:54:58,212 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1af2f4f50ff945aab3bc253a91f4cf20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732762496714 2024-11-28T02:54:58,213 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cb7be5fc7064cf5befc7795f0155f44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732762495962 2024-11-28T02:54:58,213 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18506f84b2804dfdb4a38f7cc103769f, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732762496714 2024-11-28T02:54:58,222 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:58,222 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/4d5618ff19b84a1599d1fe1b7e08cf45 is 50, key is test_row_0/A:col10/1732762497358/Put/seqid=0 2024-11-28T02:54:58,222 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:58,223 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/3d9bd65d3ae948319ee568fa818c99a3 is 50, key is test_row_0/B:col10/1732762497358/Put/seqid=0 2024-11-28T02:54:58,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742099_1275 (size=12815) 2024-11-28T02:54:58,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742100_1276 (size=12815) 2024-11-28T02:54:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T02:54:58,266 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:54:58,266 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:54:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-28T02:54:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T02:54:58,268 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:54:58,268 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:54:58,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:54:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T02:54:58,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T02:54:58,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:58,421 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T02:54:58,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:58,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:58,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:58,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:58,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:58,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:54:58,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/452afde4ecf44fe7bd3f2c89b786b852 is 50, key is test_row_0/A:col10/1732762497386/Put/seqid=0 2024-11-28T02:54:58,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742101_1277 (size=12301) 2024-11-28T02:54:58,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:54:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T02:54:58,633 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/4d5618ff19b84a1599d1fe1b7e08cf45 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4d5618ff19b84a1599d1fe1b7e08cf45 2024-11-28T02:54:58,639 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/A of 45a4746c6a5e848452a1dd3654f61765 into 4d5618ff19b84a1599d1fe1b7e08cf45(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:58,639 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:58,639 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/A, priority=13, startTime=1732762498210; duration=0sec 2024-11-28T02:54:58,639 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:54:58,639 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:A 2024-11-28T02:54:58,639 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:54:58,641 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:54:58,641 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/C is initiating minor compaction (all files) 2024-11-28T02:54:58,641 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/C in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:58,641 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/af044a4ed42e4a0c8e56ad3ea0369a9b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/941416e5e2af45cda69112234e4495fa, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/932ce8bf0417475194ba7df919bda35a] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=36.1 K 2024-11-28T02:54:58,642 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/3d9bd65d3ae948319ee568fa818c99a3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/3d9bd65d3ae948319ee568fa818c99a3 2024-11-28T02:54:58,642 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting af044a4ed42e4a0c8e56ad3ea0369a9b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732762494812 2024-11-28T02:54:58,642 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 941416e5e2af45cda69112234e4495fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732762495962 2024-11-28T02:54:58,644 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 932ce8bf0417475194ba7df919bda35a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732762496714 2024-11-28T02:54:58,649 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/B of 45a4746c6a5e848452a1dd3654f61765 into 3d9bd65d3ae948319ee568fa818c99a3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:58,649 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:58,649 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/B, priority=13, startTime=1732762498210; duration=0sec 2024-11-28T02:54:58,649 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:58,649 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:B 2024-11-28T02:54:58,653 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#C#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:54:58,654 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/8b731d5c9b8648f48a38ed0f04a53a9d is 50, key is test_row_0/C:col10/1732762497358/Put/seqid=0 2024-11-28T02:54:58,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762498609, max timestamp=1732762498651Num uniques 859; 2024-11-28T02:54:58,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762498609, max timestamp=1732762498651Num uniques 859; 2024-11-28T02:54:58,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:58,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:58,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762498609, max timestamp=1732762498651Num uniques 859; 2024-11-28T02:54:58,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:54:58,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742102_1278 (size=12815) 2024-11-28T02:54:58,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762558739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:58,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:58,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:58,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:58,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762558806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:58,809 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:54:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:58,828 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/452afde4ecf44fe7bd3f2c89b786b852 2024-11-28T02:54:58,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/9049d7a764ff483e972459e8b319d544 is 50, key is test_row_0/B:col10/1732762497386/Put/seqid=0 2024-11-28T02:54:58,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742103_1279 (size=12301) 2024-11-28T02:54:58,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762558846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T02:54:59,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:59,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762559056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:59,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:59,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:59,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:59,087 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/8b731d5c9b8648f48a38ed0f04a53a9d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/8b731d5c9b8648f48a38ed0f04a53a9d 2024-11-28T02:54:59,090 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/C of 45a4746c6a5e848452a1dd3654f61765 into 8b731d5c9b8648f48a38ed0f04a53a9d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:54:59,091 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:59,091 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/C, priority=13, startTime=1732762498210; duration=0sec 2024-11-28T02:54:59,091 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:54:59,091 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:C 2024-11-28T02:54:59,239 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/9049d7a764ff483e972459e8b319d544 2024-11-28T02:54:59,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/24aa1a6e14e7480f81808ca03aa55caa is 50, key is test_row_0/C:col10/1732762497386/Put/seqid=0 2024-11-28T02:54:59,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742104_1280 (size=12301) 2024-11-28T02:54:59,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762559366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:59,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:59,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:59,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:54:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T02:54:59,650 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/24aa1a6e14e7480f81808ca03aa55caa 2024-11-28T02:54:59,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/452afde4ecf44fe7bd3f2c89b786b852 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/452afde4ecf44fe7bd3f2c89b786b852 2024-11-28T02:54:59,657 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/452afde4ecf44fe7bd3f2c89b786b852, entries=150, sequenceid=269, filesize=12.0 K 2024-11-28T02:54:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/9049d7a764ff483e972459e8b319d544 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/9049d7a764ff483e972459e8b319d544 2024-11-28T02:54:59,662 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/9049d7a764ff483e972459e8b319d544, entries=150, sequenceid=269, filesize=12.0 K 2024-11-28T02:54:59,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/24aa1a6e14e7480f81808ca03aa55caa as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/24aa1a6e14e7480f81808ca03aa55caa 2024-11-28T02:54:59,666 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/24aa1a6e14e7480f81808ca03aa55caa, entries=150, sequenceid=269, filesize=12.0 K 2024-11-28T02:54:59,667 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 45a4746c6a5e848452a1dd3654f61765 in 1247ms, sequenceid=269, compaction requested=false 2024-11-28T02:54:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:54:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:54:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-28T02:54:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-28T02:54:59,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-28T02:54:59,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-11-28T02:54:59,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4040 sec 2024-11-28T02:54:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:54:59,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-28T02:54:59,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:54:59,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T02:54:59,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:54:59,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T02:54:59,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:54:59,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T02:54:59,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/31ba5652f6fb442e80fc14239d0dd6cb is 50, key is test_row_0/A:col10/1732762498730/Put/seqid=0 2024-11-28T02:54:59,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742105_1281 (size=14741) 2024-11-28T02:54:59,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:54:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762559914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:54:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:54:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:00,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762560026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:00,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:00,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:00,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762560236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:00,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/31ba5652f6fb442e80fc14239d0dd6cb 2024-11-28T02:55:00,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/e099268fd83e47e5869477d32b0ec614 is 50, key is test_row_0/B:col10/1732762498730/Put/seqid=0 2024-11-28T02:55:00,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742106_1282 (size=12301) 2024-11-28T02:55:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T02:55:00,416 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:00,417 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-28T02:55:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T02:55:00,418 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:00,419 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:00,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T02:55:00,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762560546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:00,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:00,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:00,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:00,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-28T02:55:00,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:00,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:55:00,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:00,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:00,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:00,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/e099268fd83e47e5869477d32b0ec614 2024-11-28T02:55:00,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/7a56a1cc7240464db14df840f057b57d is 50, key is test_row_0/C:col10/1732762498730/Put/seqid=0 2024-11-28T02:55:00,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742107_1283 (size=12301) 2024-11-28T02:55:00,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-28T02:55:00,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:00,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:55:00,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:00,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:00,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T02:55:00,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762560825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:00,829 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:00,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-28T02:55:00,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:00,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:55:00,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:00,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:00,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:01,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-28T02:55:01,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:01,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:55:01,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:01,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:01,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T02:55:01,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762561056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:01,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/7a56a1cc7240464db14df840f057b57d 2024-11-28T02:55:01,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/31ba5652f6fb442e80fc14239d0dd6cb as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/31ba5652f6fb442e80fc14239d0dd6cb 2024-11-28T02:55:01,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/31ba5652f6fb442e80fc14239d0dd6cb, entries=200, sequenceid=310, filesize=14.4 K 2024-11-28T02:55:01,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/e099268fd83e47e5869477d32b0ec614 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e099268fd83e47e5869477d32b0ec614 2024-11-28T02:55:01,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e099268fd83e47e5869477d32b0ec614, entries=150, sequenceid=310, filesize=12.0 K 2024-11-28T02:55:01,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/7a56a1cc7240464db14df840f057b57d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/7a56a1cc7240464db14df840f057b57d 2024-11-28T02:55:01,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/7a56a1cc7240464db14df840f057b57d, entries=150, sequenceid=310, filesize=12.0 K 2024-11-28T02:55:01,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 45a4746c6a5e848452a1dd3654f61765 in 1253ms, sequenceid=310, compaction requested=true 2024-11-28T02:55:01,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:01,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:01,133 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:01,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:01,133 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:01,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:01,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:01,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:01,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:01,135 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:01,135 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:01,135 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/B is initiating minor compaction (all files) 2024-11-28T02:55:01,135 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/A is initiating minor compaction (all files) 2024-11-28T02:55:01,135 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/A in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:01,135 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/B in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:01,135 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4d5618ff19b84a1599d1fe1b7e08cf45, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/452afde4ecf44fe7bd3f2c89b786b852, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/31ba5652f6fb442e80fc14239d0dd6cb] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=38.9 K 2024-11-28T02:55:01,135 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/3d9bd65d3ae948319ee568fa818c99a3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/9049d7a764ff483e972459e8b319d544, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e099268fd83e47e5869477d32b0ec614] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=36.5 K 2024-11-28T02:55:01,136 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d9bd65d3ae948319ee568fa818c99a3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732762496714 2024-11-28T02:55:01,136 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d5618ff19b84a1599d1fe1b7e08cf45, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732762496714 2024-11-28T02:55:01,136 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9049d7a764ff483e972459e8b319d544, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732762497380 2024-11-28T02:55:01,136 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 452afde4ecf44fe7bd3f2c89b786b852, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732762497380 2024-11-28T02:55:01,136 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting e099268fd83e47e5869477d32b0ec614, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732762498730 2024-11-28T02:55:01,136 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31ba5652f6fb442e80fc14239d0dd6cb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732762498679 2024-11-28T02:55:01,148 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:01,148 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/05f7a12f4a2e487997e2b66a7b30b6d9 is 50, key is test_row_0/B:col10/1732762498730/Put/seqid=0 2024-11-28T02:55:01,153 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:01,153 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/b60d5309c156413dafc35b347ccd3e73 is 50, key is test_row_0/A:col10/1732762498730/Put/seqid=0 2024-11-28T02:55:01,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742108_1284 (size=13017) 2024-11-28T02:55:01,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742109_1285 (size=13017) 2024-11-28T02:55:01,180 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/b60d5309c156413dafc35b347ccd3e73 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/b60d5309c156413dafc35b347ccd3e73 2024-11-28T02:55:01,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-28T02:55:01,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:01,182 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-28T02:55:01,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:55:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:55:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:55:01,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:01,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/02206ad4fe464dd78e43a49cf76b4e9f is 50, key is test_row_0/A:col10/1732762499908/Put/seqid=0 2024-11-28T02:55:01,189 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/A of 45a4746c6a5e848452a1dd3654f61765 into b60d5309c156413dafc35b347ccd3e73(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:01,189 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:01,190 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/A, priority=13, startTime=1732762501133; duration=0sec 2024-11-28T02:55:01,190 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:01,190 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:A 2024-11-28T02:55:01,190 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:01,192 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:01,192 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/C is initiating minor compaction (all files) 2024-11-28T02:55:01,192 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/C in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:01,192 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/8b731d5c9b8648f48a38ed0f04a53a9d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/24aa1a6e14e7480f81808ca03aa55caa, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/7a56a1cc7240464db14df840f057b57d] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=36.5 K 2024-11-28T02:55:01,192 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b731d5c9b8648f48a38ed0f04a53a9d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732762496714 2024-11-28T02:55:01,193 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24aa1a6e14e7480f81808ca03aa55caa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732762497380 2024-11-28T02:55:01,193 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a56a1cc7240464db14df840f057b57d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732762498730 2024-11-28T02:55:01,207 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:01,208 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/41d0168410274fa18cd3d733950987e6 is 50, key is test_row_0/C:col10/1732762498730/Put/seqid=0 2024-11-28T02:55:01,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742110_1286 (size=12301) 2024-11-28T02:55:01,211 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/02206ad4fe464dd78e43a49cf76b4e9f 2024-11-28T02:55:01,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/e0ecca43e24b4e43a016058ff5bf045b is 50, key is test_row_0/B:col10/1732762499908/Put/seqid=0 2024-11-28T02:55:01,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742111_1287 (size=13017) 2024-11-28T02:55:01,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742112_1288 (size=12301) 2024-11-28T02:55:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T02:55:01,569 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/05f7a12f4a2e487997e2b66a7b30b6d9 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/05f7a12f4a2e487997e2b66a7b30b6d9 2024-11-28T02:55:01,573 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/B of 45a4746c6a5e848452a1dd3654f61765 into 05f7a12f4a2e487997e2b66a7b30b6d9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:01,573 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:01,573 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/B, priority=13, startTime=1732762501133; duration=0sec 2024-11-28T02:55:01,573 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:01,573 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:B 2024-11-28T02:55:01,634 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/41d0168410274fa18cd3d733950987e6 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/41d0168410274fa18cd3d733950987e6 2024-11-28T02:55:01,638 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/C of 45a4746c6a5e848452a1dd3654f61765 into 41d0168410274fa18cd3d733950987e6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:01,638 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:01,638 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/C, priority=13, startTime=1732762501133; duration=0sec 2024-11-28T02:55:01,638 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:01,638 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:C 2024-11-28T02:55:01,653 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/e0ecca43e24b4e43a016058ff5bf045b 2024-11-28T02:55:01,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/7f6ea24a560a476d94a41cb9bd8d1b92 is 50, key is test_row_0/C:col10/1732762499908/Put/seqid=0 2024-11-28T02:55:01,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742113_1289 (size=12301) 2024-11-28T02:55:02,063 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/7f6ea24a560a476d94a41cb9bd8d1b92 2024-11-28T02:55:02,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/02206ad4fe464dd78e43a49cf76b4e9f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/02206ad4fe464dd78e43a49cf76b4e9f 2024-11-28T02:55:02,071 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/02206ad4fe464dd78e43a49cf76b4e9f, entries=150, sequenceid=319, filesize=12.0 K 2024-11-28T02:55:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/e0ecca43e24b4e43a016058ff5bf045b as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e0ecca43e24b4e43a016058ff5bf045b 2024-11-28T02:55:02,076 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e0ecca43e24b4e43a016058ff5bf045b, entries=150, sequenceid=319, filesize=12.0 K 2024-11-28T02:55:02,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/7f6ea24a560a476d94a41cb9bd8d1b92 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/7f6ea24a560a476d94a41cb9bd8d1b92 2024-11-28T02:55:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:55:02,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:55:02,087 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/7f6ea24a560a476d94a41cb9bd8d1b92, entries=150, sequenceid=319, filesize=12.0 K 2024-11-28T02:55:02,087 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=26.84 KB/27480 for 45a4746c6a5e848452a1dd3654f61765 in 905ms, sequenceid=319, compaction requested=false 2024-11-28T02:55:02,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:02,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:02,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-28T02:55:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-28T02:55:02,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-28T02:55:02,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6700 sec 2024-11-28T02:55:02,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.6740 sec 2024-11-28T02:55:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:55:02,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T02:55:02,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:55:02,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:02,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:55:02,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:02,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:55:02,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:02,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/19e6b0cd0f874161bfe5bd106d679620 is 50, key is test_row_0/A:col10/1732762502081/Put/seqid=0 2024-11-28T02:55:02,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742114_1290 (size=14741) 2024-11-28T02:55:02,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762502160, max timestamp=1732762502184Num uniques 859; 2024-11-28T02:55:02,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762502160, max timestamp=1732762502184Num uniques 859; 2024-11-28T02:55:02,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:55:02,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:55:02,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762502160, max timestamp=1732762502184Num uniques 859; 2024-11-28T02:55:02,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:55:02,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762562271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:02,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762562386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:02,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:02,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:02,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:02,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/19e6b0cd0f874161bfe5bd106d679620 2024-11-28T02:55:02,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/8376dc562f454af4980371a52b78bd54 is 50, key is test_row_0/B:col10/1732762502081/Put/seqid=0 2024-11-28T02:55:02,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742115_1291 (size=12301) 2024-11-28T02:55:02,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/8376dc562f454af4980371a52b78bd54 2024-11-28T02:55:02,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/1baa899c4adc4a91b2fe27200d753560 is 50, key is test_row_0/C:col10/1732762502081/Put/seqid=0 2024-11-28T02:55:02,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742116_1292 (size=12301) 2024-11-28T02:55:02,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T02:55:02,566 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:02,567 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-28T02:55:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T02:55:02,569 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:02,569 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:02,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:02,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762562596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:02,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:02,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:02,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T02:55:02,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T02:55:02,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:02,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:55:02,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:02,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:02,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:02,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T02:55:02,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:02,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:55:02,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:02,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:02,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T02:55:02,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762562906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:02,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:02,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/1baa899c4adc4a91b2fe27200d753560 2024-11-28T02:55:02,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/19e6b0cd0f874161bfe5bd106d679620 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/19e6b0cd0f874161bfe5bd106d679620 2024-11-28T02:55:02,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/19e6b0cd0f874161bfe5bd106d679620, entries=200, sequenceid=332, filesize=14.4 K 2024-11-28T02:55:02,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/8376dc562f454af4980371a52b78bd54 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8376dc562f454af4980371a52b78bd54 2024-11-28T02:55:02,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8376dc562f454af4980371a52b78bd54, entries=150, sequenceid=332, filesize=12.0 K 2024-11-28T02:55:02,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/1baa899c4adc4a91b2fe27200d753560 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/1baa899c4adc4a91b2fe27200d753560 2024-11-28T02:55:02,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/1baa899c4adc4a91b2fe27200d753560, entries=150, sequenceid=332, filesize=12.0 K 2024-11-28T02:55:02,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 45a4746c6a5e848452a1dd3654f61765 in 849ms, sequenceid=332, compaction requested=true 2024-11-28T02:55:02,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:02,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:02,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:02,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:02,943 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:02,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:02,943 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:02,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:02,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:02,944 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:02,944 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:02,944 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/A is initiating minor compaction (all files) 2024-11-28T02:55:02,944 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/B is initiating minor compaction (all files) 2024-11-28T02:55:02,944 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/B in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:02,944 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/A in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:02,944 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/05f7a12f4a2e487997e2b66a7b30b6d9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e0ecca43e24b4e43a016058ff5bf045b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8376dc562f454af4980371a52b78bd54] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=36.7 K 2024-11-28T02:55:02,944 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/b60d5309c156413dafc35b347ccd3e73, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/02206ad4fe464dd78e43a49cf76b4e9f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/19e6b0cd0f874161bfe5bd106d679620] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=39.1 K 2024-11-28T02:55:02,944 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b60d5309c156413dafc35b347ccd3e73, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732762498730 2024-11-28T02:55:02,944 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 05f7a12f4a2e487997e2b66a7b30b6d9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732762498730 2024-11-28T02:55:02,944 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02206ad4fe464dd78e43a49cf76b4e9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732762499898 2024-11-28T02:55:02,944 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting e0ecca43e24b4e43a016058ff5bf045b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732762499898 2024-11-28T02:55:02,945 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8376dc562f454af4980371a52b78bd54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732762502081 2024-11-28T02:55:02,945 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19e6b0cd0f874161bfe5bd106d679620, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732762502081 2024-11-28T02:55:02,958 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:02,958 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#A#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:02,958 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/184e5471bf634b6fb8377e78e7c113d2 is 50, key is test_row_0/B:col10/1732762502081/Put/seqid=0 2024-11-28T02:55:02,958 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/e6f118b03c334f44a3effb16ca6a1dc1 is 50, key is test_row_0/A:col10/1732762502081/Put/seqid=0 2024-11-28T02:55:02,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742117_1293 (size=13119) 2024-11-28T02:55:02,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742118_1294 (size=13119) 2024-11-28T02:55:03,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T02:55:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:03,028 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-28T02:55:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:55:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:55:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:55:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:03,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/49482742e1654cc09b9d474fd6c17505 is 50, key is test_row_0/A:col10/1732762502233/Put/seqid=0 2024-11-28T02:55:03,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742119_1295 (size=12301) 2024-11-28T02:55:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T02:55:03,384 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/184e5471bf634b6fb8377e78e7c113d2 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/184e5471bf634b6fb8377e78e7c113d2 2024-11-28T02:55:03,384 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/e6f118b03c334f44a3effb16ca6a1dc1 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/e6f118b03c334f44a3effb16ca6a1dc1 2024-11-28T02:55:03,388 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/A of 45a4746c6a5e848452a1dd3654f61765 into e6f118b03c334f44a3effb16ca6a1dc1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:03,388 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/B of 45a4746c6a5e848452a1dd3654f61765 into 184e5471bf634b6fb8377e78e7c113d2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:03,388 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:03,388 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:03,388 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/B, priority=13, startTime=1732762502943; duration=0sec 2024-11-28T02:55:03,388 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/A, priority=13, startTime=1732762502942; duration=0sec 2024-11-28T02:55:03,388 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:03,388 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:B 2024-11-28T02:55:03,388 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:03,388 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:A 2024-11-28T02:55:03,388 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:03,389 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:03,389 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/C is initiating minor compaction (all files) 2024-11-28T02:55:03,389 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/C in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:03,389 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/41d0168410274fa18cd3d733950987e6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/7f6ea24a560a476d94a41cb9bd8d1b92, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/1baa899c4adc4a91b2fe27200d753560] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=36.7 K 2024-11-28T02:55:03,390 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 41d0168410274fa18cd3d733950987e6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732762498730 2024-11-28T02:55:03,390 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f6ea24a560a476d94a41cb9bd8d1b92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732762499898 2024-11-28T02:55:03,390 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1baa899c4adc4a91b2fe27200d753560, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732762502081 2024-11-28T02:55:03,411 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#C#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:03,411 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/17cf2a51173046a881dfe404c3c93f3f is 50, key is test_row_0/C:col10/1732762502081/Put/seqid=0 2024-11-28T02:55:03,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742120_1296 (size=13119) 2024-11-28T02:55:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:55:03,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:55:03,437 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/49482742e1654cc09b9d474fd6c17505 2024-11-28T02:55:03,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/886a61547cf84f3db9d3bcfebb870472 is 50, key is test_row_0/B:col10/1732762502233/Put/seqid=0 2024-11-28T02:55:03,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742121_1297 (size=12301) 2024-11-28T02:55:03,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762563449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:03,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762563556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T02:55:03,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762563765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:03,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:03,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:03,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:03,820 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/17cf2a51173046a881dfe404c3c93f3f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/17cf2a51173046a881dfe404c3c93f3f 2024-11-28T02:55:03,824 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/C of 45a4746c6a5e848452a1dd3654f61765 into 17cf2a51173046a881dfe404c3c93f3f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:03,824 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:03,824 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/C, priority=13, startTime=1732762502943; duration=0sec 2024-11-28T02:55:03,824 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:03,824 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:C 2024-11-28T02:55:03,847 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/886a61547cf84f3db9d3bcfebb870472 2024-11-28T02:55:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/c0badd1f0e10417eba6f468a7a98dc0e is 50, key is test_row_0/C:col10/1732762502233/Put/seqid=0 2024-11-28T02:55:03,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742122_1298 (size=12301) 2024-11-28T02:55:04,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762564076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:04,259 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/c0badd1f0e10417eba6f468a7a98dc0e 2024-11-28T02:55:04,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/49482742e1654cc09b9d474fd6c17505 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/49482742e1654cc09b9d474fd6c17505 2024-11-28T02:55:04,269 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/49482742e1654cc09b9d474fd6c17505, entries=150, sequenceid=369, filesize=12.0 K 2024-11-28T02:55:04,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/886a61547cf84f3db9d3bcfebb870472 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/886a61547cf84f3db9d3bcfebb870472 2024-11-28T02:55:04,274 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/886a61547cf84f3db9d3bcfebb870472, entries=150, sequenceid=369, filesize=12.0 K 2024-11-28T02:55:04,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/c0badd1f0e10417eba6f468a7a98dc0e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/c0badd1f0e10417eba6f468a7a98dc0e 2024-11-28T02:55:04,279 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/c0badd1f0e10417eba6f468a7a98dc0e, entries=150, sequenceid=369, filesize=12.0 K 2024-11-28T02:55:04,280 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 45a4746c6a5e848452a1dd3654f61765 in 1252ms, sequenceid=369, compaction requested=false 2024-11-28T02:55:04,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:04,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:04,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-28T02:55:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-28T02:55:04,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-28T02:55:04,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7120 sec 2024-11-28T02:55:04,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.7170 sec 2024-11-28T02:55:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:55:04,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:55:04,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:55:04,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:04,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:55:04,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:04,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:55:04,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:04,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/d48c3131c71d4ea5a71bc631474c5b40 is 50, key is test_row_0/A:col10/1732762503447/Put/seqid=0 2024-11-28T02:55:04,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742123_1299 (size=14741) 2024-11-28T02:55:04,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/d48c3131c71d4ea5a71bc631474c5b40 2024-11-28T02:55:04,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/d29e36f4adf549578197636494bb6860 is 50, key is test_row_0/B:col10/1732762503447/Put/seqid=0 2024-11-28T02:55:04,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742124_1300 (size=12301) 2024-11-28T02:55:04,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762504650, max timestamp=1732762504665Num uniques 859; 2024-11-28T02:55:04,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762504650, max timestamp=1732762504665Num uniques 859; 2024-11-28T02:55:04,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762504650, max timestamp=1732762504665Num uniques 859; 2024-11-28T02:55:04,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:55:04,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:55:04,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:55:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T02:55:04,716 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:04,716 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-28T02:55:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T02:55:04,718 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:04,718 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:04,718 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:04,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762564743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:04,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T02:55:04,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:04,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43880 deadline: 1732762564846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:04,848 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:04,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:04,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:04,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:04,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:04,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762564856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:04,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:04,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:04,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:04,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T02:55:04,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:04,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:55:04,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:04,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:04,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:04,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43896 deadline: 1732762564986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:04,990 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:05,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/d29e36f4adf549578197636494bb6860 2024-11-28T02:55:05,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/f3c73a10c34b4513b4d949d90648276f is 50, key is test_row_0/C:col10/1732762503447/Put/seqid=0 2024-11-28T02:55:05,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742125_1301 (size=12301) 2024-11-28T02:55:05,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T02:55:05,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:05,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:55:05,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:05,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:05,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:05,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43912 deadline: 1732762565025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:05,029 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T02:55:05,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43874 deadline: 1732762565066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:05,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762565066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:05,070 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_1' in 45a4746c6a5e848452a1dd3654f61765 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18298 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:05,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T02:55:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:55:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:05,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:05,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:05,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T02:55:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:55:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:05,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T02:55:05,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762565376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:05,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/f3c73a10c34b4513b4d949d90648276f 2024-11-28T02:55:05,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/d48c3131c71d4ea5a71bc631474c5b40 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/d48c3131c71d4ea5a71bc631474c5b40 2024-11-28T02:55:05,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/d48c3131c71d4ea5a71bc631474c5b40, entries=200, sequenceid=383, filesize=14.4 K 2024-11-28T02:55:05,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/d29e36f4adf549578197636494bb6860 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d29e36f4adf549578197636494bb6860 2024-11-28T02:55:05,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d29e36f4adf549578197636494bb6860, entries=150, sequenceid=383, filesize=12.0 K 2024-11-28T02:55:05,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/f3c73a10c34b4513b4d949d90648276f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/f3c73a10c34b4513b4d949d90648276f 2024-11-28T02:55:05,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/f3c73a10c34b4513b4d949d90648276f, entries=150, sequenceid=383, filesize=12.0 K 2024-11-28T02:55:05,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 45a4746c6a5e848452a1dd3654f61765 in 852ms, sequenceid=383, compaction requested=true 2024-11-28T02:55:05,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:05,439 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:05,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:05,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:05,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:05,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:05,439 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:05,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:05,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:05,440 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:05,440 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:05,440 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/A is initiating minor compaction (all files) 2024-11-28T02:55:05,440 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/B is initiating minor compaction (all files) 2024-11-28T02:55:05,440 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/A in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:05,440 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/B in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:05,440 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/184e5471bf634b6fb8377e78e7c113d2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/886a61547cf84f3db9d3bcfebb870472, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d29e36f4adf549578197636494bb6860] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=36.8 K 2024-11-28T02:55:05,440 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/e6f118b03c334f44a3effb16ca6a1dc1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/49482742e1654cc09b9d474fd6c17505, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/d48c3131c71d4ea5a71bc631474c5b40] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=39.2 K 2024-11-28T02:55:05,441 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f118b03c334f44a3effb16ca6a1dc1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732762502081 2024-11-28T02:55:05,441 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 184e5471bf634b6fb8377e78e7c113d2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732762502081 2024-11-28T02:55:05,441 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49482742e1654cc09b9d474fd6c17505, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732762502233 2024-11-28T02:55:05,441 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 886a61547cf84f3db9d3bcfebb870472, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732762502233 2024-11-28T02:55:05,441 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d48c3131c71d4ea5a71bc631474c5b40, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732762503424 2024-11-28T02:55:05,441 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting d29e36f4adf549578197636494bb6860, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732762503437 2024-11-28T02:55:05,447 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:05,447 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/8eeb3ab439fd43e1ba63170dd249b719 is 50, key is test_row_0/B:col10/1732762503447/Put/seqid=0 2024-11-28T02:55:05,448 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#A#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:05,448 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/38a3037695d0483f838f9713fc4b2396 is 50, key is test_row_0/A:col10/1732762503447/Put/seqid=0 2024-11-28T02:55:05,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742126_1302 (size=13221) 2024-11-28T02:55:05,456 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/8eeb3ab439fd43e1ba63170dd249b719 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8eeb3ab439fd43e1ba63170dd249b719 2024-11-28T02:55:05,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742127_1303 (size=13221) 2024-11-28T02:55:05,463 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/B of 45a4746c6a5e848452a1dd3654f61765 into 8eeb3ab439fd43e1ba63170dd249b719(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:05,463 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:05,463 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/B, priority=13, startTime=1732762505439; duration=0sec 2024-11-28T02:55:05,463 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:05,463 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:B 2024-11-28T02:55:05,463 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:05,464 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:05,464 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/C is initiating minor compaction (all files) 2024-11-28T02:55:05,464 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/C in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:05,464 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/17cf2a51173046a881dfe404c3c93f3f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/c0badd1f0e10417eba6f468a7a98dc0e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/f3c73a10c34b4513b4d949d90648276f] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=36.8 K 2024-11-28T02:55:05,465 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 17cf2a51173046a881dfe404c3c93f3f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732762502081 2024-11-28T02:55:05,466 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting c0badd1f0e10417eba6f468a7a98dc0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732762502233 2024-11-28T02:55:05,466 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting f3c73a10c34b4513b4d949d90648276f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732762503437 2024-11-28T02:55:05,472 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:05,473 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/50e068bf3f7e4382a5e2f6eaed046bc5 is 50, key is test_row_0/C:col10/1732762503447/Put/seqid=0 2024-11-28T02:55:05,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T02:55:05,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:05,481 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-28T02:55:05,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:55:05,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:05,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:55:05,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:05,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:55:05,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:05,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/2b078937be9647f2b768369ec837e4f5 is 50, key is test_row_0/A:col10/1732762504720/Put/seqid=0 2024-11-28T02:55:05,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742128_1304 (size=13221) 2024-11-28T02:55:05,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742129_1305 (size=12301) 2024-11-28T02:55:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T02:55:05,862 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/38a3037695d0483f838f9713fc4b2396 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/38a3037695d0483f838f9713fc4b2396 2024-11-28T02:55:05,866 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/A of 45a4746c6a5e848452a1dd3654f61765 into 38a3037695d0483f838f9713fc4b2396(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:05,866 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:05,866 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/A, priority=13, startTime=1732762505439; duration=0sec 2024-11-28T02:55:05,866 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:05,866 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:A 2024-11-28T02:55:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:55:05,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. as already flushing 2024-11-28T02:55:05,910 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/50e068bf3f7e4382a5e2f6eaed046bc5 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/50e068bf3f7e4382a5e2f6eaed046bc5 2024-11-28T02:55:05,912 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/2b078937be9647f2b768369ec837e4f5 2024-11-28T02:55:05,915 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/C of 45a4746c6a5e848452a1dd3654f61765 into 50e068bf3f7e4382a5e2f6eaed046bc5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:05,915 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:05,915 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/C, priority=13, startTime=1732762505439; duration=0sec 2024-11-28T02:55:05,915 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:05,915 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:C 2024-11-28T02:55:05,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/e6b5b96532fc4fa8881aae678560bc96 is 50, key is test_row_0/B:col10/1732762504720/Put/seqid=0 2024-11-28T02:55:05,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742130_1306 (size=12301) 2024-11-28T02:55:05,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762565932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:05,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:05,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:05,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:06,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762566046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:06,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762566255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:06,329 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/e6b5b96532fc4fa8881aae678560bc96 2024-11-28T02:55:06,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/3bb071a7c1f24887845f1e28059b84b8 is 50, key is test_row_0/C:col10/1732762504720/Put/seqid=0 2024-11-28T02:55:06,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742131_1307 (size=12301) 2024-11-28T02:55:06,376 INFO [Thread-1052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1052. 2024-11-28T02:55:06,376 DEBUG [Thread-1052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:06,376 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:06,376 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:06,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:06,378 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-11-28T02:55:06,378 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:06,378 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:06,379 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:06,379 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:06,379 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-11-28T02:55:06,379 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:06,379 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:06,379 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:06,379 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:06,381 INFO [Thread-1060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1060. 2024-11-28T02:55:06,381 DEBUG [Thread-1060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:06,381 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:06,381 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:06,381 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:06,383 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-11-28T02:55:06,383 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:06,383 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:06,383 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:06,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:06,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43864 deadline: 1732762566566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:06,740 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/3bb071a7c1f24887845f1e28059b84b8 2024-11-28T02:55:06,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/2b078937be9647f2b768369ec837e4f5 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/2b078937be9647f2b768369ec837e4f5 2024-11-28T02:55:06,756 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/2b078937be9647f2b768369ec837e4f5, entries=150, sequenceid=420, filesize=12.0 K 2024-11-28T02:55:06,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/e6b5b96532fc4fa8881aae678560bc96 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e6b5b96532fc4fa8881aae678560bc96 2024-11-28T02:55:06,761 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e6b5b96532fc4fa8881aae678560bc96, entries=150, sequenceid=420, filesize=12.0 K 2024-11-28T02:55:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/3bb071a7c1f24887845f1e28059b84b8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3bb071a7c1f24887845f1e28059b84b8 2024-11-28T02:55:06,766 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3bb071a7c1f24887845f1e28059b84b8, entries=150, sequenceid=420, filesize=12.0 K 2024-11-28T02:55:06,767 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 45a4746c6a5e848452a1dd3654f61765 in 1286ms, sequenceid=420, compaction requested=false 2024-11-28T02:55:06,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:06,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:06,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-28T02:55:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-28T02:55:06,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-28T02:55:06,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0520 sec 2024-11-28T02:55:06,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.0570 sec 2024-11-28T02:55:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T02:55:06,878 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:55:07,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T02:55:07,079 INFO [Thread-1041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1041. 2024-11-28T02:55:07,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:55:07,080 DEBUG [Thread-1041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:07,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:07,080 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:07,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:55:07,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:07,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:55:07,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:07,080 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:07,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:07,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/954a2f314b524a73afc377085fddf04b is 50, key is test_row_0/A:col10/1732762505905/Put/seqid=0 2024-11-28T02:55:07,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742132_1308 (size=12301) 2024-11-28T02:55:07,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/954a2f314b524a73afc377085fddf04b 2024-11-28T02:55:07,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/7e7266f0141b4ef88a33484272743ed0 is 50, key is test_row_0/B:col10/1732762505905/Put/seqid=0 2024-11-28T02:55:07,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742133_1309 (size=12301) 2024-11-28T02:55:07,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/7e7266f0141b4ef88a33484272743ed0 2024-11-28T02:55:07,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/35334977f5ca479c9f2f7d02a4476d62 is 50, key is test_row_0/C:col10/1732762505905/Put/seqid=0 2024-11-28T02:55:07,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742134_1310 (size=12301) 2024-11-28T02:55:07,997 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T02:55:08,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/35334977f5ca479c9f2f7d02a4476d62 2024-11-28T02:55:08,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/954a2f314b524a73afc377085fddf04b as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/954a2f314b524a73afc377085fddf04b 2024-11-28T02:55:08,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/954a2f314b524a73afc377085fddf04b, entries=150, sequenceid=434, filesize=12.0 K 2024-11-28T02:55:08,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/7e7266f0141b4ef88a33484272743ed0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/7e7266f0141b4ef88a33484272743ed0 2024-11-28T02:55:08,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/7e7266f0141b4ef88a33484272743ed0, entries=150, sequenceid=434, filesize=12.0 K 2024-11-28T02:55:08,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/35334977f5ca479c9f2f7d02a4476d62 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/35334977f5ca479c9f2f7d02a4476d62 2024-11-28T02:55:08,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/35334977f5ca479c9f2f7d02a4476d62, entries=150, sequenceid=434, filesize=12.0 K 2024-11-28T02:55:08,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 45a4746c6a5e848452a1dd3654f61765 in 1264ms, sequenceid=434, compaction requested=true 2024-11-28T02:55:08,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:08,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:08,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:08,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:08,344 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:08,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:08,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45a4746c6a5e848452a1dd3654f61765:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:08,344 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:08,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:08,345 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:08,345 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:08,345 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/A is initiating minor compaction (all files) 2024-11-28T02:55:08,345 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/B is initiating minor compaction (all files) 2024-11-28T02:55:08,345 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/A in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:08,345 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/B in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:08,345 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/38a3037695d0483f838f9713fc4b2396, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/2b078937be9647f2b768369ec837e4f5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/954a2f314b524a73afc377085fddf04b] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=36.9 K 2024-11-28T02:55:08,345 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8eeb3ab439fd43e1ba63170dd249b719, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e6b5b96532fc4fa8881aae678560bc96, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/7e7266f0141b4ef88a33484272743ed0] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=36.9 K 2024-11-28T02:55:08,345 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eeb3ab439fd43e1ba63170dd249b719, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732762503437 2024-11-28T02:55:08,345 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38a3037695d0483f838f9713fc4b2396, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732762503437 2024-11-28T02:55:08,346 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting e6b5b96532fc4fa8881aae678560bc96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732762504714 2024-11-28T02:55:08,346 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b078937be9647f2b768369ec837e4f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732762504714 2024-11-28T02:55:08,346 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e7266f0141b4ef88a33484272743ed0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732762505905 2024-11-28T02:55:08,346 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 954a2f314b524a73afc377085fddf04b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732762505905 2024-11-28T02:55:08,355 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:08,355 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:08,356 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/a7e17bbae2d3415d9280875760e94177 is 50, key is test_row_0/B:col10/1732762505905/Put/seqid=0 2024-11-28T02:55:08,356 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/3b8981efc5134c2180ca9a6d8ba0ede5 is 50, key is test_row_0/A:col10/1732762505905/Put/seqid=0 2024-11-28T02:55:08,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742135_1311 (size=13323) 2024-11-28T02:55:08,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742136_1312 (size=13323) 2024-11-28T02:55:08,770 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/a7e17bbae2d3415d9280875760e94177 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/a7e17bbae2d3415d9280875760e94177 2024-11-28T02:55:08,770 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/3b8981efc5134c2180ca9a6d8ba0ede5 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/3b8981efc5134c2180ca9a6d8ba0ede5 2024-11-28T02:55:08,776 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/B of 45a4746c6a5e848452a1dd3654f61765 into a7e17bbae2d3415d9280875760e94177(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:08,776 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/A of 45a4746c6a5e848452a1dd3654f61765 into 3b8981efc5134c2180ca9a6d8ba0ede5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:08,776 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:08,776 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:08,776 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/B, priority=13, startTime=1732762508344; duration=0sec 2024-11-28T02:55:08,776 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/A, priority=13, startTime=1732762508343; duration=0sec 2024-11-28T02:55:08,776 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:08,776 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:B 2024-11-28T02:55:08,776 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:08,776 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:A 2024-11-28T02:55:08,776 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:08,777 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:08,777 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 45a4746c6a5e848452a1dd3654f61765/C is initiating minor compaction (all files) 2024-11-28T02:55:08,777 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45a4746c6a5e848452a1dd3654f61765/C in TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:08,777 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/50e068bf3f7e4382a5e2f6eaed046bc5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3bb071a7c1f24887845f1e28059b84b8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/35334977f5ca479c9f2f7d02a4476d62] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp, totalSize=36.9 K 2024-11-28T02:55:08,777 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 50e068bf3f7e4382a5e2f6eaed046bc5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732762503437 2024-11-28T02:55:08,778 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bb071a7c1f24887845f1e28059b84b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732762504714 2024-11-28T02:55:08,778 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 35334977f5ca479c9f2f7d02a4476d62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732762505905 2024-11-28T02:55:08,784 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45a4746c6a5e848452a1dd3654f61765#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:08,785 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/dd632d23873c4f5cb68a4bf306d99953 is 50, key is test_row_0/C:col10/1732762505905/Put/seqid=0 2024-11-28T02:55:08,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742137_1313 (size=13323) 2024-11-28T02:55:09,193 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/dd632d23873c4f5cb68a4bf306d99953 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/dd632d23873c4f5cb68a4bf306d99953 2024-11-28T02:55:09,198 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45a4746c6a5e848452a1dd3654f61765/C of 45a4746c6a5e848452a1dd3654f61765 into dd632d23873c4f5cb68a4bf306d99953(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:09,198 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45a4746c6a5e848452a1dd3654f61765: 2024-11-28T02:55:09,198 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., storeName=45a4746c6a5e848452a1dd3654f61765/C, priority=13, startTime=1732762508344; duration=0sec 2024-11-28T02:55:09,198 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:09,198 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45a4746c6a5e848452a1dd3654f61765:C 2024-11-28T02:55:14,950 INFO [Thread-1049 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1049. 2024-11-28T02:55:14,950 DEBUG [Thread-1049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:14,950 DEBUG [Thread-1049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:14,950 DEBUG [Thread-1049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:14,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:15,031 INFO [Thread-1047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1047. 2024-11-28T02:55:15,031 DEBUG [Thread-1047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:15,031 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:15,031 DEBUG [Thread-1047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:15,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:15,079 INFO [Thread-1045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1045. 2024-11-28T02:55:15,079 DEBUG [Thread-1045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:15,079 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:15,079 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:15,079 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:15,149 INFO [Thread-1043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1043. 2024-11-28T02:55:15,149 DEBUG [Thread-1043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:15,149 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:15,149 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:15,149 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:15,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T02:55:15,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 236 2024-11-28T02:55:15,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-28T02:55:15,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-28T02:55:15,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-28T02:55:15,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-28T02:55:15,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T02:55:15,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3486 2024-11-28T02:55:15,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3240 2024-11-28T02:55:15,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3132 2024-11-28T02:55:15,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3485 2024-11-28T02:55:15,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3256 2024-11-28T02:55:15,150 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T02:55:15,151 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:55:15,151 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:55:15,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:15,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:15,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:15,153 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T02:55:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T02:55:15,158 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762515158"}]},"ts":"1732762515158"} 2024-11-28T02:55:15,160 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T02:55:15,160 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T02:55:15,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T02:55:15,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45a4746c6a5e848452a1dd3654f61765, UNASSIGN}] 2024-11-28T02:55:15,164 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45a4746c6a5e848452a1dd3654f61765, UNASSIGN 2024-11-28T02:55:15,165 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=45a4746c6a5e848452a1dd3654f61765, regionState=CLOSING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:55:15,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45a4746c6a5e848452a1dd3654f61765, UNASSIGN because future has completed 2024-11-28T02:55:15,167 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T02:55:15,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:55:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T02:55:15,320 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:55:15,320 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T02:55:15,321 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 45a4746c6a5e848452a1dd3654f61765, disabling compactions & flushes 2024-11-28T02:55:15,321 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:15,321 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:15,321 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. after waiting 0 ms 2024-11-28T02:55:15,321 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:15,321 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing 45a4746c6a5e848452a1dd3654f61765 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-28T02:55:15,321 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=A 2024-11-28T02:55:15,321 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:15,321 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=B 2024-11-28T02:55:15,321 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:15,321 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45a4746c6a5e848452a1dd3654f61765, store=C 2024-11-28T02:55:15,322 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:15,326 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/04b6acdc0213469f87b73e414468aaf7 is 50, key is test_row_0/A:col10/1732762515027/Put/seqid=0 2024-11-28T02:55:15,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742138_1314 (size=12301) 2024-11-28T02:55:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T02:55:15,731 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/04b6acdc0213469f87b73e414468aaf7 2024-11-28T02:55:15,744 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/a2f0858e42e64de3b99e1198f1c0ae7d is 50, key is test_row_0/B:col10/1732762515027/Put/seqid=0 2024-11-28T02:55:15,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742139_1315 (size=12301) 2024-11-28T02:55:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T02:55:16,149 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/a2f0858e42e64de3b99e1198f1c0ae7d 2024-11-28T02:55:16,163 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/3c9558e3670b411294274fe72d4668a0 is 50, key is test_row_0/C:col10/1732762515027/Put/seqid=0 2024-11-28T02:55:16,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742140_1316 (size=12301) 2024-11-28T02:55:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T02:55:16,569 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/3c9558e3670b411294274fe72d4668a0 2024-11-28T02:55:16,579 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/A/04b6acdc0213469f87b73e414468aaf7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/04b6acdc0213469f87b73e414468aaf7 2024-11-28T02:55:16,585 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/04b6acdc0213469f87b73e414468aaf7, entries=150, sequenceid=444, filesize=12.0 K 2024-11-28T02:55:16,586 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/B/a2f0858e42e64de3b99e1198f1c0ae7d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/a2f0858e42e64de3b99e1198f1c0ae7d 2024-11-28T02:55:16,592 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/a2f0858e42e64de3b99e1198f1c0ae7d, entries=150, sequenceid=444, filesize=12.0 K 2024-11-28T02:55:16,593 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/.tmp/C/3c9558e3670b411294274fe72d4668a0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3c9558e3670b411294274fe72d4668a0 2024-11-28T02:55:16,598 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3c9558e3670b411294274fe72d4668a0, entries=150, sequenceid=444, filesize=12.0 K 2024-11-28T02:55:16,599 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 45a4746c6a5e848452a1dd3654f61765 in 1278ms, sequenceid=444, compaction requested=false 2024-11-28T02:55:16,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/5c84fcf1416a403d9b5bc7d897f31982, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/a13e87e55d3e4ff79d22318e96c0f838, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/492d03859895401cbea23e3b301b76c8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4a6a99a7e750459fb15be6ee3754035c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/6557f22d6f47437c91c5378b076f1ec3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4978275cde2c4ab0a6c87a624b24b7ae, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/bfacfd9341494639ba67afbe757a3f22, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/0763c00e878946ee9b1c62506afc81e8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/2e5ee20f54e8473191fb1694c3b4f6c0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/ec92e339a88844f9937282cf5297e46f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/eafa880ee55c4a1c966f0a31adb6e91a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/de035d84b36449b499bae80a4eb86578, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/91639a1741ae4f4d9bc5f51398ee9b36, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/1c04af4536d84620897af5e72f3bbb0a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/95c144c4fe0d4e49b75fbb0a30be1e28, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/f8002d7059db45d5aace1d22af12935a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/8cb7be5fc7064cf5befc7795f0155f44, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/18506f84b2804dfdb4a38f7cc103769f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4d5618ff19b84a1599d1fe1b7e08cf45, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/452afde4ecf44fe7bd3f2c89b786b852, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/31ba5652f6fb442e80fc14239d0dd6cb, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/b60d5309c156413dafc35b347ccd3e73, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/02206ad4fe464dd78e43a49cf76b4e9f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/19e6b0cd0f874161bfe5bd106d679620, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/e6f118b03c334f44a3effb16ca6a1dc1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/49482742e1654cc09b9d474fd6c17505, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/d48c3131c71d4ea5a71bc631474c5b40, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/38a3037695d0483f838f9713fc4b2396, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/2b078937be9647f2b768369ec837e4f5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/954a2f314b524a73afc377085fddf04b] to archive 2024-11-28T02:55:16,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:55:16,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/5c84fcf1416a403d9b5bc7d897f31982 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/5c84fcf1416a403d9b5bc7d897f31982 2024-11-28T02:55:16,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/a13e87e55d3e4ff79d22318e96c0f838 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/a13e87e55d3e4ff79d22318e96c0f838 2024-11-28T02:55:16,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/492d03859895401cbea23e3b301b76c8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/492d03859895401cbea23e3b301b76c8 2024-11-28T02:55:16,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4a6a99a7e750459fb15be6ee3754035c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4a6a99a7e750459fb15be6ee3754035c 2024-11-28T02:55:16,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/6557f22d6f47437c91c5378b076f1ec3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/6557f22d6f47437c91c5378b076f1ec3 2024-11-28T02:55:16,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4978275cde2c4ab0a6c87a624b24b7ae to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4978275cde2c4ab0a6c87a624b24b7ae 2024-11-28T02:55:16,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/bfacfd9341494639ba67afbe757a3f22 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/bfacfd9341494639ba67afbe757a3f22 2024-11-28T02:55:16,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/0763c00e878946ee9b1c62506afc81e8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/0763c00e878946ee9b1c62506afc81e8 2024-11-28T02:55:16,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/2e5ee20f54e8473191fb1694c3b4f6c0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/2e5ee20f54e8473191fb1694c3b4f6c0 2024-11-28T02:55:16,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/ec92e339a88844f9937282cf5297e46f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/ec92e339a88844f9937282cf5297e46f 2024-11-28T02:55:16,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/eafa880ee55c4a1c966f0a31adb6e91a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/eafa880ee55c4a1c966f0a31adb6e91a 2024-11-28T02:55:16,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/de035d84b36449b499bae80a4eb86578 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/de035d84b36449b499bae80a4eb86578 2024-11-28T02:55:16,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/91639a1741ae4f4d9bc5f51398ee9b36 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/91639a1741ae4f4d9bc5f51398ee9b36 2024-11-28T02:55:16,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/1c04af4536d84620897af5e72f3bbb0a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/1c04af4536d84620897af5e72f3bbb0a 2024-11-28T02:55:16,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/95c144c4fe0d4e49b75fbb0a30be1e28 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/95c144c4fe0d4e49b75fbb0a30be1e28 2024-11-28T02:55:16,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/f8002d7059db45d5aace1d22af12935a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/f8002d7059db45d5aace1d22af12935a 2024-11-28T02:55:16,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/8cb7be5fc7064cf5befc7795f0155f44 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/8cb7be5fc7064cf5befc7795f0155f44 2024-11-28T02:55:16,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/18506f84b2804dfdb4a38f7cc103769f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/18506f84b2804dfdb4a38f7cc103769f 2024-11-28T02:55:16,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4d5618ff19b84a1599d1fe1b7e08cf45 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/4d5618ff19b84a1599d1fe1b7e08cf45 2024-11-28T02:55:16,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/452afde4ecf44fe7bd3f2c89b786b852 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/452afde4ecf44fe7bd3f2c89b786b852 2024-11-28T02:55:16,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/31ba5652f6fb442e80fc14239d0dd6cb to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/31ba5652f6fb442e80fc14239d0dd6cb 2024-11-28T02:55:16,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/b60d5309c156413dafc35b347ccd3e73 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/b60d5309c156413dafc35b347ccd3e73 2024-11-28T02:55:16,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/02206ad4fe464dd78e43a49cf76b4e9f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/02206ad4fe464dd78e43a49cf76b4e9f 2024-11-28T02:55:16,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/19e6b0cd0f874161bfe5bd106d679620 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/19e6b0cd0f874161bfe5bd106d679620 2024-11-28T02:55:16,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/e6f118b03c334f44a3effb16ca6a1dc1 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/e6f118b03c334f44a3effb16ca6a1dc1 2024-11-28T02:55:16,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/49482742e1654cc09b9d474fd6c17505 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/49482742e1654cc09b9d474fd6c17505 2024-11-28T02:55:16,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/d48c3131c71d4ea5a71bc631474c5b40 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/d48c3131c71d4ea5a71bc631474c5b40 2024-11-28T02:55:16,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/38a3037695d0483f838f9713fc4b2396 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/38a3037695d0483f838f9713fc4b2396 2024-11-28T02:55:16,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/2b078937be9647f2b768369ec837e4f5 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/2b078937be9647f2b768369ec837e4f5 2024-11-28T02:55:16,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/954a2f314b524a73afc377085fddf04b to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/954a2f314b524a73afc377085fddf04b 2024-11-28T02:55:16,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d8bd4f2d212b42cc83781eda99d9c3a0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/fda4358e5e0b4a7d84f9d5fc9bfbe5e2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/c929117a7c764f4281578075410013ef, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6a4e1b5e4ef041378f35d1aa8d72fc7a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6252c83a43d4444d92192dc7832e677c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/5cab72e4d0a2411d8ce9fe3deb256e56, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/1f963928244d426bb1c8680a7ba1eb44, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6ef2588c30d24e4fb15e87e242756466, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/5d04e689971443c7b6521dc7fda9091a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d76b33e54bd542a185201e4c5baad7b0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/a0b80cbdf3984e1aa68c80ab6a4a665a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8f50c3f9667944449d2f3b09ad94b3e4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/68ebe7f6aa24485885ee4802429e0ab1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/4d8da61c03554b2ba203990caf739ad8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d7ea00d481304eb4a40bf7d8ff3027e2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/f1ba7e41e10b4b708a3de86a42e1c678, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/712110eb485b48bda2d1fcb6f1a515b7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/3d9bd65d3ae948319ee568fa818c99a3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/1af2f4f50ff945aab3bc253a91f4cf20, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/9049d7a764ff483e972459e8b319d544, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/05f7a12f4a2e487997e2b66a7b30b6d9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e099268fd83e47e5869477d32b0ec614, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e0ecca43e24b4e43a016058ff5bf045b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/184e5471bf634b6fb8377e78e7c113d2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8376dc562f454af4980371a52b78bd54, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/886a61547cf84f3db9d3bcfebb870472, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8eeb3ab439fd43e1ba63170dd249b719, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d29e36f4adf549578197636494bb6860, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e6b5b96532fc4fa8881aae678560bc96, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/7e7266f0141b4ef88a33484272743ed0] to archive 2024-11-28T02:55:16,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:55:16,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d8bd4f2d212b42cc83781eda99d9c3a0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d8bd4f2d212b42cc83781eda99d9c3a0 2024-11-28T02:55:16,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/fda4358e5e0b4a7d84f9d5fc9bfbe5e2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/fda4358e5e0b4a7d84f9d5fc9bfbe5e2 2024-11-28T02:55:16,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/c929117a7c764f4281578075410013ef to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/c929117a7c764f4281578075410013ef 2024-11-28T02:55:16,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6a4e1b5e4ef041378f35d1aa8d72fc7a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6a4e1b5e4ef041378f35d1aa8d72fc7a 2024-11-28T02:55:16,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6252c83a43d4444d92192dc7832e677c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6252c83a43d4444d92192dc7832e677c 2024-11-28T02:55:16,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/5cab72e4d0a2411d8ce9fe3deb256e56 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/5cab72e4d0a2411d8ce9fe3deb256e56 2024-11-28T02:55:16,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/1f963928244d426bb1c8680a7ba1eb44 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/1f963928244d426bb1c8680a7ba1eb44 2024-11-28T02:55:16,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6ef2588c30d24e4fb15e87e242756466 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/6ef2588c30d24e4fb15e87e242756466 2024-11-28T02:55:16,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/5d04e689971443c7b6521dc7fda9091a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/5d04e689971443c7b6521dc7fda9091a 2024-11-28T02:55:16,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d76b33e54bd542a185201e4c5baad7b0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d76b33e54bd542a185201e4c5baad7b0 2024-11-28T02:55:16,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/a0b80cbdf3984e1aa68c80ab6a4a665a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/a0b80cbdf3984e1aa68c80ab6a4a665a 2024-11-28T02:55:16,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8f50c3f9667944449d2f3b09ad94b3e4 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8f50c3f9667944449d2f3b09ad94b3e4 2024-11-28T02:55:16,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/68ebe7f6aa24485885ee4802429e0ab1 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/68ebe7f6aa24485885ee4802429e0ab1 2024-11-28T02:55:16,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/4d8da61c03554b2ba203990caf739ad8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/4d8da61c03554b2ba203990caf739ad8 2024-11-28T02:55:16,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d7ea00d481304eb4a40bf7d8ff3027e2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d7ea00d481304eb4a40bf7d8ff3027e2 2024-11-28T02:55:16,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/f1ba7e41e10b4b708a3de86a42e1c678 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/f1ba7e41e10b4b708a3de86a42e1c678 2024-11-28T02:55:16,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/712110eb485b48bda2d1fcb6f1a515b7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/712110eb485b48bda2d1fcb6f1a515b7 2024-11-28T02:55:16,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/3d9bd65d3ae948319ee568fa818c99a3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/3d9bd65d3ae948319ee568fa818c99a3 2024-11-28T02:55:16,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/1af2f4f50ff945aab3bc253a91f4cf20 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/1af2f4f50ff945aab3bc253a91f4cf20 2024-11-28T02:55:16,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/9049d7a764ff483e972459e8b319d544 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/9049d7a764ff483e972459e8b319d544 2024-11-28T02:55:16,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/05f7a12f4a2e487997e2b66a7b30b6d9 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/05f7a12f4a2e487997e2b66a7b30b6d9 2024-11-28T02:55:16,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e099268fd83e47e5869477d32b0ec614 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e099268fd83e47e5869477d32b0ec614 2024-11-28T02:55:16,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e0ecca43e24b4e43a016058ff5bf045b to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e0ecca43e24b4e43a016058ff5bf045b 2024-11-28T02:55:16,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/184e5471bf634b6fb8377e78e7c113d2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/184e5471bf634b6fb8377e78e7c113d2 2024-11-28T02:55:16,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8376dc562f454af4980371a52b78bd54 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8376dc562f454af4980371a52b78bd54 2024-11-28T02:55:16,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/886a61547cf84f3db9d3bcfebb870472 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/886a61547cf84f3db9d3bcfebb870472 2024-11-28T02:55:16,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8eeb3ab439fd43e1ba63170dd249b719 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/8eeb3ab439fd43e1ba63170dd249b719 2024-11-28T02:55:16,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d29e36f4adf549578197636494bb6860 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/d29e36f4adf549578197636494bb6860 2024-11-28T02:55:16,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e6b5b96532fc4fa8881aae678560bc96 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/e6b5b96532fc4fa8881aae678560bc96 2024-11-28T02:55:16,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/7e7266f0141b4ef88a33484272743ed0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/7e7266f0141b4ef88a33484272743ed0 2024-11-28T02:55:16,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/2f4e3d35935f4bf5b2a723ea9271ea5b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/9dce4bce31f9448e98f4d3ed2e1402f7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/360fb0c4c38f4978bbd744efb634d51d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/8c6db4c7af464cf19480fdfea2bbbbda, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/db9c974558a846f2bbc5e972963dcdc6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/bffd413fb1754055a1f6123c6de07ff1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/079cf441d6cf494c96a988cdb3f1a3d9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/489c91bb198b4031bef013b61cbed788, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3c02923568964ecca263f46037061d18, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/10254d6541f74271b1f8a4961599da86, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/2f027c2368e548d38641d7f0050459c2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/d81d0a92b59841b7ab2a3c2e3149c7e6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/ab6d6709a9ce47a3903efb5a49a0d4ec, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/ce78c72f93c543f8a5b5438bf31b730e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/af044a4ed42e4a0c8e56ad3ea0369a9b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/78f671a23d8040ea8466686d00d00b81, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/941416e5e2af45cda69112234e4495fa, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/8b731d5c9b8648f48a38ed0f04a53a9d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/932ce8bf0417475194ba7df919bda35a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/24aa1a6e14e7480f81808ca03aa55caa, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/41d0168410274fa18cd3d733950987e6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/7a56a1cc7240464db14df840f057b57d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/7f6ea24a560a476d94a41cb9bd8d1b92, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/17cf2a51173046a881dfe404c3c93f3f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/1baa899c4adc4a91b2fe27200d753560, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/c0badd1f0e10417eba6f468a7a98dc0e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/50e068bf3f7e4382a5e2f6eaed046bc5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/f3c73a10c34b4513b4d949d90648276f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3bb071a7c1f24887845f1e28059b84b8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/35334977f5ca479c9f2f7d02a4476d62] to archive 2024-11-28T02:55:16,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:55:16,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/2f4e3d35935f4bf5b2a723ea9271ea5b to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/2f4e3d35935f4bf5b2a723ea9271ea5b 2024-11-28T02:55:16,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/9dce4bce31f9448e98f4d3ed2e1402f7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/9dce4bce31f9448e98f4d3ed2e1402f7 2024-11-28T02:55:16,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/360fb0c4c38f4978bbd744efb634d51d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/360fb0c4c38f4978bbd744efb634d51d 2024-11-28T02:55:16,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/8c6db4c7af464cf19480fdfea2bbbbda to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/8c6db4c7af464cf19480fdfea2bbbbda 2024-11-28T02:55:16,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/db9c974558a846f2bbc5e972963dcdc6 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/db9c974558a846f2bbc5e972963dcdc6 2024-11-28T02:55:16,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/bffd413fb1754055a1f6123c6de07ff1 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/bffd413fb1754055a1f6123c6de07ff1 2024-11-28T02:55:16,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/079cf441d6cf494c96a988cdb3f1a3d9 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/079cf441d6cf494c96a988cdb3f1a3d9 2024-11-28T02:55:16,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/489c91bb198b4031bef013b61cbed788 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/489c91bb198b4031bef013b61cbed788 2024-11-28T02:55:16,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3c02923568964ecca263f46037061d18 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3c02923568964ecca263f46037061d18 2024-11-28T02:55:16,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/10254d6541f74271b1f8a4961599da86 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/10254d6541f74271b1f8a4961599da86 2024-11-28T02:55:16,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/2f027c2368e548d38641d7f0050459c2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/2f027c2368e548d38641d7f0050459c2 2024-11-28T02:55:16,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/d81d0a92b59841b7ab2a3c2e3149c7e6 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/d81d0a92b59841b7ab2a3c2e3149c7e6 2024-11-28T02:55:16,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/ab6d6709a9ce47a3903efb5a49a0d4ec to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/ab6d6709a9ce47a3903efb5a49a0d4ec 2024-11-28T02:55:16,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/ce78c72f93c543f8a5b5438bf31b730e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/ce78c72f93c543f8a5b5438bf31b730e 2024-11-28T02:55:16,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/af044a4ed42e4a0c8e56ad3ea0369a9b to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/af044a4ed42e4a0c8e56ad3ea0369a9b 2024-11-28T02:55:16,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/78f671a23d8040ea8466686d00d00b81 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/78f671a23d8040ea8466686d00d00b81 2024-11-28T02:55:16,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/941416e5e2af45cda69112234e4495fa to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/941416e5e2af45cda69112234e4495fa 2024-11-28T02:55:16,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/8b731d5c9b8648f48a38ed0f04a53a9d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/8b731d5c9b8648f48a38ed0f04a53a9d 2024-11-28T02:55:16,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/932ce8bf0417475194ba7df919bda35a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/932ce8bf0417475194ba7df919bda35a 2024-11-28T02:55:16,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/24aa1a6e14e7480f81808ca03aa55caa to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/24aa1a6e14e7480f81808ca03aa55caa 2024-11-28T02:55:16,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/41d0168410274fa18cd3d733950987e6 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/41d0168410274fa18cd3d733950987e6 2024-11-28T02:55:16,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/7a56a1cc7240464db14df840f057b57d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/7a56a1cc7240464db14df840f057b57d 2024-11-28T02:55:16,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/7f6ea24a560a476d94a41cb9bd8d1b92 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/7f6ea24a560a476d94a41cb9bd8d1b92 2024-11-28T02:55:16,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/17cf2a51173046a881dfe404c3c93f3f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/17cf2a51173046a881dfe404c3c93f3f 2024-11-28T02:55:16,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/1baa899c4adc4a91b2fe27200d753560 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/1baa899c4adc4a91b2fe27200d753560 2024-11-28T02:55:16,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/c0badd1f0e10417eba6f468a7a98dc0e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/c0badd1f0e10417eba6f468a7a98dc0e 2024-11-28T02:55:16,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/50e068bf3f7e4382a5e2f6eaed046bc5 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/50e068bf3f7e4382a5e2f6eaed046bc5 2024-11-28T02:55:16,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/f3c73a10c34b4513b4d949d90648276f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/f3c73a10c34b4513b4d949d90648276f 2024-11-28T02:55:16,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3bb071a7c1f24887845f1e28059b84b8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3bb071a7c1f24887845f1e28059b84b8 2024-11-28T02:55:16,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/35334977f5ca479c9f2f7d02a4476d62 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/35334977f5ca479c9f2f7d02a4476d62 2024-11-28T02:55:16,685 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/recovered.edits/447.seqid, newMaxSeqId=447, maxSeqId=1 2024-11-28T02:55:16,686 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. 2024-11-28T02:55:16,686 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 45a4746c6a5e848452a1dd3654f61765: Waiting for close lock at 1732762515321Running coprocessor pre-close hooks at 1732762515321Disabling compacts and flushes for region at 1732762515321Disabling writes for close at 1732762515321Obtaining lock to block concurrent updates at 1732762515321Preparing flush snapshotting stores in 45a4746c6a5e848452a1dd3654f61765 at 1732762515321Finished memstore snapshotting TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732762515322 (+1 ms)Flushing stores of TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765. at 1732762515322Flushing 45a4746c6a5e848452a1dd3654f61765/A: creating writer at 1732762515322Flushing 45a4746c6a5e848452a1dd3654f61765/A: appending metadata at 1732762515325 (+3 ms)Flushing 45a4746c6a5e848452a1dd3654f61765/A: closing flushed file at 1732762515325Flushing 45a4746c6a5e848452a1dd3654f61765/B: creating writer at 1732762515741 (+416 ms)Flushing 45a4746c6a5e848452a1dd3654f61765/B: appending metadata at 1732762515743 (+2 ms)Flushing 45a4746c6a5e848452a1dd3654f61765/B: closing flushed file at 1732762515743Flushing 45a4746c6a5e848452a1dd3654f61765/C: creating writer at 1732762516159 (+416 ms)Flushing 45a4746c6a5e848452a1dd3654f61765/C: appending metadata at 1732762516162 (+3 ms)Flushing 45a4746c6a5e848452a1dd3654f61765/C: closing flushed file at 1732762516162Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bbc7a0d: reopening flushed file at 1732762516577 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@244b874c: reopening flushed file at 1732762516585 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38ad1479: reopening flushed file at 1732762516592 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 45a4746c6a5e848452a1dd3654f61765 in 1278ms, sequenceid=444, compaction requested=false at 1732762516599 (+7 ms)Writing region close event to WAL at 1732762516683 (+84 ms)Running coprocessor post-close hooks at 1732762516686 (+3 ms)Closed at 1732762516686 2024-11-28T02:55:16,688 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:55:16,688 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=45a4746c6a5e848452a1dd3654f61765, regionState=CLOSED 2024-11-28T02:55:16,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:55:16,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-28T02:55:16,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 45a4746c6a5e848452a1dd3654f61765, server=978bd30859c3,41631,1732762419946 in 1.5230 sec 2024-11-28T02:55:16,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-28T02:55:16,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45a4746c6a5e848452a1dd3654f61765, UNASSIGN in 1.5290 sec 2024-11-28T02:55:16,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-28T02:55:16,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5330 sec 2024-11-28T02:55:16,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762516696"}]},"ts":"1732762516696"} 2024-11-28T02:55:16,697 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T02:55:16,697 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T02:55:16,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5450 sec 2024-11-28T02:55:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T02:55:17,307 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:17,308 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T02:55:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:17,311 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-28T02:55:17,313 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:17,316 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:55:17,320 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/recovered.edits] 2024-11-28T02:55:17,323 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/04b6acdc0213469f87b73e414468aaf7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/04b6acdc0213469f87b73e414468aaf7 2024-11-28T02:55:17,324 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/3b8981efc5134c2180ca9a6d8ba0ede5 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/A/3b8981efc5134c2180ca9a6d8ba0ede5 2024-11-28T02:55:17,326 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/a2f0858e42e64de3b99e1198f1c0ae7d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/a2f0858e42e64de3b99e1198f1c0ae7d 2024-11-28T02:55:17,327 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/a7e17bbae2d3415d9280875760e94177 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/B/a7e17bbae2d3415d9280875760e94177 2024-11-28T02:55:17,329 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3c9558e3670b411294274fe72d4668a0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/3c9558e3670b411294274fe72d4668a0 2024-11-28T02:55:17,329 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/dd632d23873c4f5cb68a4bf306d99953 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/C/dd632d23873c4f5cb68a4bf306d99953 2024-11-28T02:55:17,331 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/recovered.edits/447.seqid to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765/recovered.edits/447.seqid 2024-11-28T02:55:17,332 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/45a4746c6a5e848452a1dd3654f61765 2024-11-28T02:55:17,332 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T02:55:17,333 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:17,336 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T02:55:17,338 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T02:55:17,339 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:17,339 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T02:55:17,339 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732762517339"}]},"ts":"9223372036854775807"} 2024-11-28T02:55:17,340 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T02:55:17,340 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 45a4746c6a5e848452a1dd3654f61765, NAME => 'TestAcidGuarantees,,1732762484166.45a4746c6a5e848452a1dd3654f61765.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T02:55:17,340 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T02:55:17,340 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732762517340"}]},"ts":"9223372036854775807"} 2024-11-28T02:55:17,342 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T02:55:17,342 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:17,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-28T02:55:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-28T02:55:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T02:55:17,417 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:17,431 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=455 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=429 (was 476), ProcessCount=11 (was 11), AvailableMemoryMB=7197 (was 7450) 2024-11-28T02:55:17,443 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=429, ProcessCount=11, AvailableMemoryMB=7197 2024-11-28T02:55:17,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T02:55:17,445 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:55:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:17,447 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T02:55:17,448 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:17,448 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-28T02:55:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T02:55:17,449 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T02:55:17,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742141_1317 (size=1061) 2024-11-28T02:55:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T02:55:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T02:55:17,860 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381 2024-11-28T02:55:17,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742142_1318 (size=53) 2024-11-28T02:55:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T02:55:18,270 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:55:18,270 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d1bea096782fcdd0d3636fed7e5e84b2, disabling compactions & flushes 2024-11-28T02:55:18,270 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:18,270 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:18,270 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. after waiting 0 ms 2024-11-28T02:55:18,270 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:18,270 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:18,271 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d1bea096782fcdd0d3636fed7e5e84b2: Waiting for close lock at 1732762518270Disabling compacts and flushes for region at 1732762518270Disabling writes for close at 1732762518270Writing region close event to WAL at 1732762518270Closed at 1732762518270 2024-11-28T02:55:18,271 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T02:55:18,272 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732762518272"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732762518272"}]},"ts":"1732762518272"} 2024-11-28T02:55:18,273 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T02:55:18,274 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T02:55:18,274 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762518274"}]},"ts":"1732762518274"} 2024-11-28T02:55:18,275 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T02:55:18,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, ASSIGN}] 2024-11-28T02:55:18,276 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, ASSIGN 2024-11-28T02:55:18,277 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, ASSIGN; state=OFFLINE, location=978bd30859c3,41631,1732762419946; forceNewPlan=false, retain=false 2024-11-28T02:55:18,428 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d1bea096782fcdd0d3636fed7e5e84b2, regionState=OPENING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:55:18,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, ASSIGN because future has completed 2024-11-28T02:55:18,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:55:18,584 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:18,584 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:55:18,584 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:18,585 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:55:18,585 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:18,585 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T02:55:18,586 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:18,588 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:55:18,588 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1bea096782fcdd0d3636fed7e5e84b2 columnFamilyName A 2024-11-28T02:55:18,588 DEBUG [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:18,589 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.HStore(327): Store=d1bea096782fcdd0d3636fed7e5e84b2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:55:18,589 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:18,590 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:55:18,590 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1bea096782fcdd0d3636fed7e5e84b2 columnFamilyName B 2024-11-28T02:55:18,590 DEBUG [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:18,591 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.HStore(327): Store=d1bea096782fcdd0d3636fed7e5e84b2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:55:18,591 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:18,592 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:55:18,592 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1bea096782fcdd0d3636fed7e5e84b2 columnFamilyName C 2024-11-28T02:55:18,593 DEBUG [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:18,593 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.HStore(327): Store=d1bea096782fcdd0d3636fed7e5e84b2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:55:18,593 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:18,593 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:18,594 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:18,594 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:18,595 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:18,596 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:18,596 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T02:55:18,597 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:18,599 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:55:18,600 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened d1bea096782fcdd0d3636fed7e5e84b2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66376694, jitterRate=-0.010910183191299438}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T02:55:18,600 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:18,601 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for d1bea096782fcdd0d3636fed7e5e84b2: Running coprocessor pre-open hook at 1732762518585Writing region info on filesystem at 1732762518585Initializing all the Stores at 1732762518586 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762518586Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762518586Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762518586Cleaning up temporary data from old regions at 1732762518596 (+10 ms)Running coprocessor post-open hooks at 1732762518600 (+4 ms)Region opened successfully at 1732762518600 2024-11-28T02:55:18,601 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., pid=91, masterSystemTime=1732762518581 2024-11-28T02:55:18,603 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:18,603 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:18,604 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d1bea096782fcdd0d3636fed7e5e84b2, regionState=OPEN, openSeqNum=2, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:55:18,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:55:18,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-28T02:55:18,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 in 177 msec 2024-11-28T02:55:18,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-28T02:55:18,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, ASSIGN in 333 msec 2024-11-28T02:55:18,611 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T02:55:18,611 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762518611"}]},"ts":"1732762518611"} 2024-11-28T02:55:18,613 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T02:55:18,614 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T02:55:18,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-28T02:55:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T02:55:19,596 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:19,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f9102f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:19,598 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:19,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:19,599 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4146d1c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:19,601 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0e8299, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:19,602 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:19,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:19,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:19,605 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T02:55:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 978bd30859c3,32823,1732762419008 2024-11-28T02:55:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@578c7b66 2024-11-28T02:55:19,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T02:55:19,608 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T02:55:19,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T02:55:19,610 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:55:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:19,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742143_1319 (size=1097) 2024-11-28T02:55:20,033 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T02:55:20,033 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T02:55:20,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T02:55:20,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, REOPEN/MOVE}] 2024-11-28T02:55:20,043 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, REOPEN/MOVE 2024-11-28T02:55:20,044 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=d1bea096782fcdd0d3636fed7e5e84b2, regionState=CLOSING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:55:20,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, REOPEN/MOVE because future has completed 2024-11-28T02:55:20,046 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T02:55:20,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:55:20,199 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,199 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T02:55:20,199 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing d1bea096782fcdd0d3636fed7e5e84b2, disabling compactions & flushes 2024-11-28T02:55:20,199 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:20,199 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:20,199 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. after waiting 0 ms 2024-11-28T02:55:20,199 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:20,204 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T02:55:20,205 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:20,205 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for d1bea096782fcdd0d3636fed7e5e84b2: Waiting for close lock at 1732762520199Running coprocessor pre-close hooks at 1732762520199Disabling compacts and flushes for region at 1732762520199Disabling writes for close at 1732762520199Writing region close event to WAL at 1732762520200 (+1 ms)Running coprocessor post-close hooks at 1732762520204 (+4 ms)Closed at 1732762520205 (+1 ms) 2024-11-28T02:55:20,205 WARN [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: d1bea096782fcdd0d3636fed7e5e84b2 to self. 2024-11-28T02:55:20,207 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,208 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=d1bea096782fcdd0d3636fed7e5e84b2, regionState=CLOSED 2024-11-28T02:55:20,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:55:20,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-28T02:55:20,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 in 165 msec 2024-11-28T02:55:20,215 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, REOPEN/MOVE; state=CLOSED, location=978bd30859c3,41631,1732762419946; forceNewPlan=false, retain=true 2024-11-28T02:55:20,366 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=d1bea096782fcdd0d3636fed7e5e84b2, regionState=OPENING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:55:20,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, REOPEN/MOVE because future has completed 2024-11-28T02:55:20,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:55:20,528 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:20,528 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:55:20,529 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,529 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:55:20,529 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,529 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,531 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,533 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:55:20,534 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1bea096782fcdd0d3636fed7e5e84b2 columnFamilyName A 2024-11-28T02:55:20,536 DEBUG [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:20,537 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.HStore(327): Store=d1bea096782fcdd0d3636fed7e5e84b2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:55:20,538 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,539 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:55:20,539 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1bea096782fcdd0d3636fed7e5e84b2 columnFamilyName B 2024-11-28T02:55:20,539 DEBUG [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:20,540 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.HStore(327): Store=d1bea096782fcdd0d3636fed7e5e84b2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:55:20,540 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,540 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:55:20,541 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1bea096782fcdd0d3636fed7e5e84b2 columnFamilyName C 2024-11-28T02:55:20,541 DEBUG [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:20,541 INFO [StoreOpener-d1bea096782fcdd0d3636fed7e5e84b2-1 {}] regionserver.HStore(327): Store=d1bea096782fcdd0d3636fed7e5e84b2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:55:20,541 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:20,541 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,542 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,543 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,545 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,545 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,545 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T02:55:20,547 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,548 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened d1bea096782fcdd0d3636fed7e5e84b2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73047167, jitterRate=0.08848761022090912}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T02:55:20,548 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,548 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for d1bea096782fcdd0d3636fed7e5e84b2: Running coprocessor pre-open hook at 1732762520529Writing region info on filesystem at 1732762520529Initializing all the Stores at 1732762520530 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762520531 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762520531Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762520531Cleaning up temporary data from old regions at 1732762520545 (+14 ms)Running coprocessor post-open hooks at 1732762520548 (+3 ms)Region opened successfully at 1732762520548 2024-11-28T02:55:20,549 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., pid=96, masterSystemTime=1732762520523 2024-11-28T02:55:20,551 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:20,551 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:20,552 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=d1bea096782fcdd0d3636fed7e5e84b2, regionState=OPEN, openSeqNum=5, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:55:20,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:55:20,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-28T02:55:20,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 in 184 msec 2024-11-28T02:55:20,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-28T02:55:20,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, REOPEN/MOVE in 514 msec 2024-11-28T02:55:20,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-28T02:55:20,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 519 msec 2024-11-28T02:55:20,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 952 msec 2024-11-28T02:55:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-28T02:55:20,565 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:20,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec1958f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:20,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:20,568 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6acec4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:20,570 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:20,570 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3723a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:20,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:20,573 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37688616, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:20,575 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@640a2ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:20,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:20,578 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d0ad1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:20,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:20,579 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:20,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41417fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:20,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:20,582 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35126065, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:20,584 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c864a71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:20,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:20,587 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1220634b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:20,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:20,588 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@647d954f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:20,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:20,595 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7132e238, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:20,597 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed4f27d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,598 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:20,599 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:20,599 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:20,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388aed4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:20,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:20,601 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77bb0580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,602 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:20,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:20,603 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0c911d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:20,605 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38900c9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:20,606 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:20,607 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:20,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:20,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:20,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@243519f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:20,609 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44801b3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,610 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:20,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:20,610 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@539200a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:20,612 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:20,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca51a95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:20,613 DEBUG [Thread-1487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:20,613 DEBUG [Thread-1489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:20,613 DEBUG [Thread-1476 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:20,613 DEBUG [Thread-1485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:20,613 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:20,613 DEBUG [Thread-1483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:20,614 DEBUG [Thread-1474 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:20,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:20,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:20,614 DEBUG [Thread-1489 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:20,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:20,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:20,614 DEBUG [Thread-1489 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:20,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:20,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-28T02:55:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:20,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:20,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:20,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:20,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T02:55:20,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:20,616 DEBUG [Thread-1480 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:20,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:20,617 DEBUG [Thread-1491 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:20,617 DEBUG [Thread-1472 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:20,617 DEBUG [Thread-1478 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:20,617 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:20,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:55:20,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:55:20,618 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:20,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:20,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:20,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:20,619 DEBUG [Thread-1478 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:20,619 DEBUG [Thread-1478 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:55:20,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:20,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:20,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:20,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:55:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:55:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:55:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:55:20,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:55:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:55:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:55:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:20,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T02:55:20,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:20,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:20,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:20,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:20,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:20,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:20,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112887f44bd032414762b44b186447992e85_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762520626/Put/seqid=0 2024-11-28T02:55:20,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742144_1320 (size=12154) 2024-11-28T02:55:20,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762520656, max timestamp=1732762520668Num uniques 859; 2024-11-28T02:55:20,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:20,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762520656, max timestamp=1732762520668Num uniques 859; 2024-11-28T02:55:20,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:20,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762520656, max timestamp=1732762520671Num uniques 859; 2024-11-28T02:55:20,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:55:20,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762580699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762580706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:20,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762580708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:20,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762580708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:20,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762580714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T02:55:20,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T02:55:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:20,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:20,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762580816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:20,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762580816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:20,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762580826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:20,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762580836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762580836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:20,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T02:55:20,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:20,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:20,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:20,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:20,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T02:55:21,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762581036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762581036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762581036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762581046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762581046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,067 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:21,070 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112887f44bd032414762b44b186447992e85_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112887f44bd032414762b44b186447992e85_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:21,071 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/dbba94a6f19e45ceab892a3ece3ba722, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:21,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/dbba94a6f19e45ceab892a3ece3ba722 is 175, key is test_row_0/A:col10/1732762520626/Put/seqid=0 2024-11-28T02:55:21,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742145_1321 (size=30955) 2024-11-28T02:55:21,077 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/dbba94a6f19e45ceab892a3ece3ba722 2024-11-28T02:55:21,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T02:55:21,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:21,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:21,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:21,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/c7e6b79deeae4cedaa2911f369b20ebe is 50, key is test_row_0/B:col10/1732762520626/Put/seqid=0 2024-11-28T02:55:21,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742146_1322 (size=12001) 2024-11-28T02:55:21,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T02:55:21,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:21,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:21,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:21,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T02:55:21,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762581346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762581346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762581346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762581366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762581366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T02:55:21,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:21,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:21,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:21,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/c7e6b79deeae4cedaa2911f369b20ebe 2024-11-28T02:55:21,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/b3839927eb2f4f4ab2205ea2de67508f is 50, key is test_row_0/C:col10/1732762520626/Put/seqid=0 2024-11-28T02:55:21,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742147_1323 (size=12001) 2024-11-28T02:55:21,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T02:55:21,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:21,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:21,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:21,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T02:55:21,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:21,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:21,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:21,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T02:55:21,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T02:55:21,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:21,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:21,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:21,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:21,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762581867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762581867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762581867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762581876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762581886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:21,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/b3839927eb2f4f4ab2205ea2de67508f 2024-11-28T02:55:21,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/dbba94a6f19e45ceab892a3ece3ba722 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbba94a6f19e45ceab892a3ece3ba722 2024-11-28T02:55:21,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbba94a6f19e45ceab892a3ece3ba722, entries=150, sequenceid=15, filesize=30.2 K 2024-11-28T02:55:21,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/c7e6b79deeae4cedaa2911f369b20ebe as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/c7e6b79deeae4cedaa2911f369b20ebe 2024-11-28T02:55:21,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/c7e6b79deeae4cedaa2911f369b20ebe, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T02:55:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/b3839927eb2f4f4ab2205ea2de67508f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/b3839927eb2f4f4ab2205ea2de67508f 2024-11-28T02:55:21,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/b3839927eb2f4f4ab2205ea2de67508f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T02:55:21,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for d1bea096782fcdd0d3636fed7e5e84b2 in 1322ms, sequenceid=15, compaction requested=false 2024-11-28T02:55:21,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:22,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T02:55:22,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:22,008 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-28T02:55:22,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:22,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:22,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:22,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:22,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:22,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:22,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a205b5494e354c8193a4fa4f66b4147e_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762520705/Put/seqid=0 2024-11-28T02:55:22,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742148_1324 (size=12154) 2024-11-28T02:55:22,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,040 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a205b5494e354c8193a4fa4f66b4147e_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a205b5494e354c8193a4fa4f66b4147e_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/7ae15e26c032457a83778cfbf48ec2aa, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:22,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/7ae15e26c032457a83778cfbf48ec2aa is 175, key is test_row_0/A:col10/1732762520705/Put/seqid=0 2024-11-28T02:55:22,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742149_1325 (size=30955) 2024-11-28T02:55:22,209 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T02:55:22,454 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/7ae15e26c032457a83778cfbf48ec2aa 2024-11-28T02:55:22,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/2197ea7d18c04c69b64b8dd76686eb81 is 50, key is test_row_0/B:col10/1732762520705/Put/seqid=0 2024-11-28T02:55:22,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742150_1326 (size=12001) 2024-11-28T02:55:22,473 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/2197ea7d18c04c69b64b8dd76686eb81 2024-11-28T02:55:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/11d014bb28f0455eac62fdeef87a9ea4 is 50, key is test_row_0/C:col10/1732762520705/Put/seqid=0 2024-11-28T02:55:22,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742151_1327 (size=12001) 2024-11-28T02:55:22,487 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/11d014bb28f0455eac62fdeef87a9ea4 2024-11-28T02:55:22,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/7ae15e26c032457a83778cfbf48ec2aa as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7ae15e26c032457a83778cfbf48ec2aa 2024-11-28T02:55:22,496 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7ae15e26c032457a83778cfbf48ec2aa, entries=150, sequenceid=50, filesize=30.2 K 2024-11-28T02:55:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/2197ea7d18c04c69b64b8dd76686eb81 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/2197ea7d18c04c69b64b8dd76686eb81 2024-11-28T02:55:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,501 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/2197ea7d18c04c69b64b8dd76686eb81, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T02:55:22,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/11d014bb28f0455eac62fdeef87a9ea4 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/11d014bb28f0455eac62fdeef87a9ea4 2024-11-28T02:55:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,506 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/11d014bb28f0455eac62fdeef87a9ea4, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T02:55:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,507 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=0 B/0 for d1bea096782fcdd0d3636fed7e5e84b2 in 498ms, sequenceid=50, compaction requested=false 2024-11-28T02:55:22,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:22,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-28T02:55:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-28T02:55:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-28T02:55:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8900 sec 2024-11-28T02:55:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8980 sec 2024-11-28T02:55:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T02:55:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,766 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,766 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-28T02:55:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,769 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T02:55:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,770 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T02:55:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-28T02:55:22,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-28T02:55:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-28T02:55:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-28T02:55:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-28T02:55:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 160 msec 2024-11-28T02:55:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T02:55:22,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:22,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:22,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:22,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:22,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fe07f1122d3f4b50bf1e5ea94627bcc5_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762522960/Put/seqid=0 2024-11-28T02:55:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742152_1328 (size=21918) 2024-11-28T02:55:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762523017, max timestamp=1732762523034Num uniques 859; 2024-11-28T02:55:23,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762523017, max timestamp=1732762523034Num uniques 859; 2024-11-28T02:55:23,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:55:23,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:55:23,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762523017, max timestamp=1732762523034Num uniques 859; 2024-11-28T02:55:23,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:55:23,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762583063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762583070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762583071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T02:55:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762583072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,087 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:23,087 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-28T02:55:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T02:55:23,089 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:23,089 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:23,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:23,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762583078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T02:55:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762583186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762583186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762583186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762583195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762583206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T02:55:23,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:23,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:23,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:23,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T02:55:23,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:23,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:23,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:23,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,398 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:23,401 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fe07f1122d3f4b50bf1e5ea94627bcc5_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fe07f1122d3f4b50bf1e5ea94627bcc5_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:23,401 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/cdc5e47bab9b4c34b80128d3116cbf38, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:23,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/cdc5e47bab9b4c34b80128d3116cbf38 is 175, key is test_row_0/A:col10/1732762522960/Put/seqid=0 2024-11-28T02:55:23,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742153_1329 (size=65323) 2024-11-28T02:55:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T02:55:23,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762583406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762583406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762583406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762583406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762583416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T02:55:23,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:23,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:23,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:23,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T02:55:23,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:23,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:23,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:23,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T02:55:23,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762583726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762583726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762583726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762583731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762583736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:23,806 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/cdc5e47bab9b4c34b80128d3116cbf38 2024-11-28T02:55:23,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/5199f7a374a24911b87600e7bc7cc3c6 is 50, key is test_row_0/B:col10/1732762522960/Put/seqid=0 2024-11-28T02:55:23,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742154_1330 (size=12001) 2024-11-28T02:55:23,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T02:55:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:23,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T02:55:24,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:24,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T02:55:24,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:24,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T02:55:24,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/5199f7a374a24911b87600e7bc7cc3c6 2024-11-28T02:55:24,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762584246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:24,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762584246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:24,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762584246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:24,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762584256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:24,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:24,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:24,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:24,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/d855613a7f9d4752b49e497155a3fd56 is 50, key is test_row_0/C:col10/1732762522960/Put/seqid=0 2024-11-28T02:55:24,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762584256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:24,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742155_1331 (size=12001) 2024-11-28T02:55:24,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T02:55:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T02:55:24,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:24,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T02:55:24,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:24,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:24,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/d855613a7f9d4752b49e497155a3fd56 2024-11-28T02:55:24,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/cdc5e47bab9b4c34b80128d3116cbf38 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/cdc5e47bab9b4c34b80128d3116cbf38 2024-11-28T02:55:24,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/cdc5e47bab9b4c34b80128d3116cbf38, entries=350, sequenceid=61, filesize=63.8 K 2024-11-28T02:55:24,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/5199f7a374a24911b87600e7bc7cc3c6 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/5199f7a374a24911b87600e7bc7cc3c6 2024-11-28T02:55:24,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/5199f7a374a24911b87600e7bc7cc3c6, entries=150, sequenceid=61, filesize=11.7 K 2024-11-28T02:55:24,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/d855613a7f9d4752b49e497155a3fd56 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/d855613a7f9d4752b49e497155a3fd56 2024-11-28T02:55:24,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/d855613a7f9d4752b49e497155a3fd56, entries=150, sequenceid=61, filesize=11.7 K 2024-11-28T02:55:24,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d1bea096782fcdd0d3636fed7e5e84b2 in 1765ms, sequenceid=61, compaction requested=true 2024-11-28T02:55:24,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:24,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:24,735 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:24,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:24,735 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:24,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:24,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:24,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:24,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:24,736 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:24,737 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/A is initiating minor compaction (all files) 2024-11-28T02:55:24,737 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/A in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,737 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbba94a6f19e45ceab892a3ece3ba722, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7ae15e26c032457a83778cfbf48ec2aa, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/cdc5e47bab9b4c34b80128d3116cbf38] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=124.3 K 2024-11-28T02:55:24,737 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,737 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbba94a6f19e45ceab892a3ece3ba722, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7ae15e26c032457a83778cfbf48ec2aa, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/cdc5e47bab9b4c34b80128d3116cbf38] 2024-11-28T02:55:24,737 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:24,737 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/B is initiating minor compaction (all files) 2024-11-28T02:55:24,738 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/B in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,738 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/c7e6b79deeae4cedaa2911f369b20ebe, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/2197ea7d18c04c69b64b8dd76686eb81, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/5199f7a374a24911b87600e7bc7cc3c6] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=35.2 K 2024-11-28T02:55:24,738 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting c7e6b79deeae4cedaa2911f369b20ebe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732762520626 2024-11-28T02:55:24,738 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbba94a6f19e45ceab892a3ece3ba722, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732762520626 2024-11-28T02:55:24,739 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2197ea7d18c04c69b64b8dd76686eb81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732762520705 2024-11-28T02:55:24,739 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ae15e26c032457a83778cfbf48ec2aa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732762520705 2024-11-28T02:55:24,740 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5199f7a374a24911b87600e7bc7cc3c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732762522947 2024-11-28T02:55:24,741 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdc5e47bab9b4c34b80128d3116cbf38, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732762522943 2024-11-28T02:55:24,753 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#B#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:24,753 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/aebc85e7631c4e6f955fbfdca538cdf3 is 50, key is test_row_0/B:col10/1732762522960/Put/seqid=0 2024-11-28T02:55:24,759 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:24,768 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411280540e176923141d2a2a6a5d98852a739_d1bea096782fcdd0d3636fed7e5e84b2 store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:24,770 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411280540e176923141d2a2a6a5d98852a739_d1bea096782fcdd0d3636fed7e5e84b2, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:24,770 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280540e176923141d2a2a6a5d98852a739_d1bea096782fcdd0d3636fed7e5e84b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:24,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T02:55:24,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:24,782 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-28T02:55:24,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:24,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:24,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:24,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:24,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:24,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:24,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742156_1332 (size=12104) 2024-11-28T02:55:24,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283b15a869fb7d41f6b5887aba8cb67e3b_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762523070/Put/seqid=0 2024-11-28T02:55:24,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742157_1333 (size=4469) 2024-11-28T02:55:24,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742158_1334 (size=12154) 2024-11-28T02:55:25,203 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/aebc85e7631c4e6f955fbfdca538cdf3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/aebc85e7631c4e6f955fbfdca538cdf3 2024-11-28T02:55:25,208 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/B of d1bea096782fcdd0d3636fed7e5e84b2 into aebc85e7631c4e6f955fbfdca538cdf3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:25,208 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:25,208 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/B, priority=13, startTime=1732762524735; duration=0sec 2024-11-28T02:55:25,208 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:25,208 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:B 2024-11-28T02:55:25,208 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:25,209 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:25,209 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/C is initiating minor compaction (all files) 2024-11-28T02:55:25,209 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/C in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:25,209 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/b3839927eb2f4f4ab2205ea2de67508f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/11d014bb28f0455eac62fdeef87a9ea4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/d855613a7f9d4752b49e497155a3fd56] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=35.2 K 2024-11-28T02:55:25,210 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting b3839927eb2f4f4ab2205ea2de67508f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732762520626 2024-11-28T02:55:25,210 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 11d014bb28f0455eac62fdeef87a9ea4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732762520705 2024-11-28T02:55:25,210 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting d855613a7f9d4752b49e497155a3fd56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732762522947 2024-11-28T02:55:25,221 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:25,221 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/29c9d8d58baf4b1b8b6fa450c2e3ab20 is 50, key is test_row_0/C:col10/1732762522960/Put/seqid=0 2024-11-28T02:55:25,221 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#A#compaction#283 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:25,222 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/1241afa619654ae897c0bdbec8cdfa10 is 175, key is test_row_0/A:col10/1732762522960/Put/seqid=0 2024-11-28T02:55:25,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742159_1335 (size=12104) 2024-11-28T02:55:25,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742160_1336 (size=31058) 2024-11-28T02:55:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T02:55:25,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:25,265 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283b15a869fb7d41f6b5887aba8cb67e3b_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283b15a869fb7d41f6b5887aba8cb67e3b_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:25,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/eab5cdcef3684316ae7aaf0ec92d91c7, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:25,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/eab5cdcef3684316ae7aaf0ec92d91c7 is 175, key is test_row_0/A:col10/1732762523070/Put/seqid=0 2024-11-28T02:55:25,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742161_1337 (size=30955) 2024-11-28T02:55:25,285 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/eab5cdcef3684316ae7aaf0ec92d91c7 2024-11-28T02:55:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:25,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:25,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/07c687a2adf74f889491796c75084d38 is 50, key is test_row_0/B:col10/1732762523070/Put/seqid=0 2024-11-28T02:55:25,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762585302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762585315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762585323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762585324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762585328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742162_1338 (size=12001) 2024-11-28T02:55:25,355 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/07c687a2adf74f889491796c75084d38 2024-11-28T02:55:25,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/8da9a4870767496484fa6b84844a29f0 is 50, key is test_row_0/C:col10/1732762523070/Put/seqid=0 2024-11-28T02:55:25,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742163_1339 (size=12001) 2024-11-28T02:55:25,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762585435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762585436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762585446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762585446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762585446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,633 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/29c9d8d58baf4b1b8b6fa450c2e3ab20 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/29c9d8d58baf4b1b8b6fa450c2e3ab20 2024-11-28T02:55:25,639 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/C of d1bea096782fcdd0d3636fed7e5e84b2 into 29c9d8d58baf4b1b8b6fa450c2e3ab20(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:25,639 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:25,639 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/C, priority=13, startTime=1732762524736; duration=0sec 2024-11-28T02:55:25,640 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:25,640 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:C 2024-11-28T02:55:25,642 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/1241afa619654ae897c0bdbec8cdfa10 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/1241afa619654ae897c0bdbec8cdfa10 2024-11-28T02:55:25,648 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/A of d1bea096782fcdd0d3636fed7e5e84b2 into 1241afa619654ae897c0bdbec8cdfa10(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:25,648 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:25,648 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/A, priority=13, startTime=1732762524735; duration=0sec 2024-11-28T02:55:25,648 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:25,648 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:A 2024-11-28T02:55:25,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762585656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762585656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762585666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762585666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762585667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:25,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:25,780 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/8da9a4870767496484fa6b84844a29f0 2024-11-28T02:55:25,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/eab5cdcef3684316ae7aaf0ec92d91c7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/eab5cdcef3684316ae7aaf0ec92d91c7 2024-11-28T02:55:25,791 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/eab5cdcef3684316ae7aaf0ec92d91c7, entries=150, sequenceid=96, filesize=30.2 K 2024-11-28T02:55:25,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/07c687a2adf74f889491796c75084d38 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/07c687a2adf74f889491796c75084d38 2024-11-28T02:55:25,799 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/07c687a2adf74f889491796c75084d38, entries=150, sequenceid=96, filesize=11.7 K 2024-11-28T02:55:25,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/8da9a4870767496484fa6b84844a29f0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8da9a4870767496484fa6b84844a29f0 2024-11-28T02:55:25,812 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8da9a4870767496484fa6b84844a29f0, entries=150, sequenceid=96, filesize=11.7 K 2024-11-28T02:55:25,813 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for d1bea096782fcdd0d3636fed7e5e84b2 in 1031ms, sequenceid=96, compaction requested=false 2024-11-28T02:55:25,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:25,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:25,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-28T02:55:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-28T02:55:25,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-28T02:55:25,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7270 sec 2024-11-28T02:55:25,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.7320 sec 2024-11-28T02:55:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:25,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:55:25,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:25,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:25,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:25,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:25,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:25,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:26,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a13c047ec42e4dbd99bf673a9181e9c3_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762525326/Put/seqid=0 2024-11-28T02:55:26,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742164_1340 (size=14594) 2024-11-28T02:55:26,025 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:26,031 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a13c047ec42e4dbd99bf673a9181e9c3_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a13c047ec42e4dbd99bf673a9181e9c3_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:26,033 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/a19143f005684f0a829bd783c0ade653, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:26,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/a19143f005684f0a829bd783c0ade653 is 175, key is test_row_0/A:col10/1732762525326/Put/seqid=0 2024-11-28T02:55:26,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742165_1341 (size=39549) 2024-11-28T02:55:26,052 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/a19143f005684f0a829bd783c0ade653 2024-11-28T02:55:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762586035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762586033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762586036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732762526028, max timestamp=1732762526051Num uniques 859; 2024-11-28T02:55:26,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732762526028, max timestamp=1732762526051Num uniques 859; 2024-11-28T02:55:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762526028, max timestamp=1732762526051Num uniques 859; 2024-11-28T02:55:26,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:26,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/36de86748cd24c359bdf2670b0c6bb67 is 50, key is test_row_0/B:col10/1732762525326/Put/seqid=0 2024-11-28T02:55:26,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742166_1342 (size=12001) 2024-11-28T02:55:26,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762586147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762586153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762586166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762586166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762586166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762586265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762586276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762586386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762586386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762586386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/36de86748cd24c359bdf2670b0c6bb67 2024-11-28T02:55:26,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/42d1a4aafb524788874d6ee92b2c043c is 50, key is test_row_0/C:col10/1732762525326/Put/seqid=0 2024-11-28T02:55:26,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762586486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742167_1343 (size=12001) 2024-11-28T02:55:26,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762586496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/42d1a4aafb524788874d6ee92b2c043c 2024-11-28T02:55:26,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/a19143f005684f0a829bd783c0ade653 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/a19143f005684f0a829bd783c0ade653 2024-11-28T02:55:26,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/a19143f005684f0a829bd783c0ade653, entries=200, sequenceid=110, filesize=38.6 K 2024-11-28T02:55:26,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/36de86748cd24c359bdf2670b0c6bb67 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/36de86748cd24c359bdf2670b0c6bb67 2024-11-28T02:55:26,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/36de86748cd24c359bdf2670b0c6bb67, entries=150, sequenceid=110, filesize=11.7 K 2024-11-28T02:55:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/42d1a4aafb524788874d6ee92b2c043c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/42d1a4aafb524788874d6ee92b2c043c 2024-11-28T02:55:26,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/42d1a4aafb524788874d6ee92b2c043c, entries=150, sequenceid=110, filesize=11.7 K 2024-11-28T02:55:26,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d1bea096782fcdd0d3636fed7e5e84b2 in 556ms, sequenceid=110, compaction requested=true 2024-11-28T02:55:26,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:26,538 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:26,539 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:26,539 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/A is initiating minor compaction (all files) 2024-11-28T02:55:26,539 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/A in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:26,540 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/1241afa619654ae897c0bdbec8cdfa10, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/eab5cdcef3684316ae7aaf0ec92d91c7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/a19143f005684f0a829bd783c0ade653] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=99.2 K 2024-11-28T02:55:26,540 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:26,540 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/1241afa619654ae897c0bdbec8cdfa10, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/eab5cdcef3684316ae7aaf0ec92d91c7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/a19143f005684f0a829bd783c0ade653] 2024-11-28T02:55:26,540 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1241afa619654ae897c0bdbec8cdfa10, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732762522947 2024-11-28T02:55:26,540 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting eab5cdcef3684316ae7aaf0ec92d91c7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732762523070 2024-11-28T02:55:26,541 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a19143f005684f0a829bd783c0ade653, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762525313 2024-11-28T02:55:26,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:26,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:26,549 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:26,550 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:26,550 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/B is initiating minor compaction (all files) 2024-11-28T02:55:26,550 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/B in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:26,550 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/aebc85e7631c4e6f955fbfdca538cdf3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/07c687a2adf74f889491796c75084d38, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/36de86748cd24c359bdf2670b0c6bb67] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=35.3 K 2024-11-28T02:55:26,551 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting aebc85e7631c4e6f955fbfdca538cdf3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732762522947 2024-11-28T02:55:26,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:26,551 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 07c687a2adf74f889491796c75084d38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732762523070 2024-11-28T02:55:26,551 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 36de86748cd24c359bdf2670b0c6bb67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762525320 2024-11-28T02:55:26,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:26,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:26,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:26,567 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:26,577 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#B#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:26,577 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/149d3925077f4a3dbb7095c6173f2f26 is 50, key is test_row_0/B:col10/1732762525326/Put/seqid=0 2024-11-28T02:55:26,581 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128453c4c2dbab4482b8284080af0223e1e_d1bea096782fcdd0d3636fed7e5e84b2 store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:26,583 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128453c4c2dbab4482b8284080af0223e1e_d1bea096782fcdd0d3636fed7e5e84b2, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:26,583 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128453c4c2dbab4482b8284080af0223e1e_d1bea096782fcdd0d3636fed7e5e84b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:26,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742169_1345 (size=4469) 2024-11-28T02:55:26,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742168_1344 (size=12207) 2024-11-28T02:55:26,601 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/149d3925077f4a3dbb7095c6173f2f26 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/149d3925077f4a3dbb7095c6173f2f26 2024-11-28T02:55:26,605 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/B of d1bea096782fcdd0d3636fed7e5e84b2 into 149d3925077f4a3dbb7095c6173f2f26(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:26,605 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:26,605 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/B, priority=13, startTime=1732762526549; duration=0sec 2024-11-28T02:55:26,605 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:26,605 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:B 2024-11-28T02:55:26,606 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:26,608 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:26,608 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/C is initiating minor compaction (all files) 2024-11-28T02:55:26,608 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/C in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:26,608 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/29c9d8d58baf4b1b8b6fa450c2e3ab20, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8da9a4870767496484fa6b84844a29f0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/42d1a4aafb524788874d6ee92b2c043c] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=35.3 K 2024-11-28T02:55:26,610 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 29c9d8d58baf4b1b8b6fa450c2e3ab20, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732762522947 2024-11-28T02:55:26,610 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8da9a4870767496484fa6b84844a29f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732762523070 2024-11-28T02:55:26,611 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 42d1a4aafb524788874d6ee92b2c043c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762525320 2024-11-28T02:55:26,616 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#C#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:26,616 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/a83fab5c1aa24cfda1179ce3ccd0f20b is 50, key is test_row_0/C:col10/1732762525326/Put/seqid=0 2024-11-28T02:55:26,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742170_1346 (size=12207) 2024-11-28T02:55:26,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T02:55:26,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:26,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:26,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:26,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:26,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:26,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:26,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280541406c7ae144a3a64397df1d48996d_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762526713/Put/seqid=0 2024-11-28T02:55:26,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742171_1347 (size=14794) 2024-11-28T02:55:26,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:26,747 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280541406c7ae144a3a64397df1d48996d_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280541406c7ae144a3a64397df1d48996d_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:26,748 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/7dcaff905d70469a8f0af29bb79a7d63, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:26,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/7dcaff905d70469a8f0af29bb79a7d63 is 175, key is test_row_0/A:col10/1732762526713/Put/seqid=0 2024-11-28T02:55:26,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762586751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762586759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762586759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742172_1348 (size=39749) 2024-11-28T02:55:26,789 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/7dcaff905d70469a8f0af29bb79a7d63 2024-11-28T02:55:26,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/540e6caa07844367a7b9091359baad31 is 50, key is test_row_0/B:col10/1732762526713/Put/seqid=0 2024-11-28T02:55:26,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742173_1349 (size=12151) 2024-11-28T02:55:26,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/540e6caa07844367a7b9091359baad31 2024-11-28T02:55:26,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762586816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762586816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/8217ac0310e04dd3ab293ee6434f8205 is 50, key is test_row_0/C:col10/1732762526713/Put/seqid=0 2024-11-28T02:55:26,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742174_1350 (size=12151) 2024-11-28T02:55:26,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/8217ac0310e04dd3ab293ee6434f8205 2024-11-28T02:55:26,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/7dcaff905d70469a8f0af29bb79a7d63 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7dcaff905d70469a8f0af29bb79a7d63 2024-11-28T02:55:26,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7dcaff905d70469a8f0af29bb79a7d63, entries=200, sequenceid=147, filesize=38.8 K 2024-11-28T02:55:26,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/540e6caa07844367a7b9091359baad31 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/540e6caa07844367a7b9091359baad31 2024-11-28T02:55:26,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/540e6caa07844367a7b9091359baad31, entries=150, sequenceid=147, filesize=11.9 K 2024-11-28T02:55:26,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/8217ac0310e04dd3ab293ee6434f8205 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8217ac0310e04dd3ab293ee6434f8205 2024-11-28T02:55:26,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:26,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8217ac0310e04dd3ab293ee6434f8205, entries=150, sequenceid=147, filesize=11.9 K 2024-11-28T02:55:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762586866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:26,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:26,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:26,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for d1bea096782fcdd0d3636fed7e5e84b2 in 164ms, sequenceid=147, compaction requested=false 2024-11-28T02:55:26,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:26,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T02:55:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:26,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:26,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:26,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:26,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:26,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:26,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:26,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c13bb4c7bebd4c73a87b05a64e0a5b57_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762526749/Put/seqid=0 2024-11-28T02:55:26,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742175_1351 (size=17284) 2024-11-28T02:55:26,991 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#A#compaction#291 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:26,992 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/3ae9d17168134cceaafd2069a1484c9a is 175, key is test_row_0/A:col10/1732762525326/Put/seqid=0 2024-11-28T02:55:27,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742176_1352 (size=31161) 2024-11-28T02:55:27,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762586993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762587005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,041 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/a83fab5c1aa24cfda1179ce3ccd0f20b as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/a83fab5c1aa24cfda1179ce3ccd0f20b 2024-11-28T02:55:27,047 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/C of d1bea096782fcdd0d3636fed7e5e84b2 into a83fab5c1aa24cfda1179ce3ccd0f20b(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:27,047 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:27,047 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/C, priority=13, startTime=1732762526553; duration=0sec 2024-11-28T02:55:27,047 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:27,047 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:C 2024-11-28T02:55:27,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762587086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762587116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762587116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T02:55:27,256 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:27,256 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-28T02:55:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T02:55:27,258 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:27,258 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:27,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:27,316 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:27,320 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c13bb4c7bebd4c73a87b05a64e0a5b57_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c13bb4c7bebd4c73a87b05a64e0a5b57_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:27,321 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/ccdd1ab569a942098be82e4b85ee2880, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:27,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/ccdd1ab569a942098be82e4b85ee2880 is 175, key is test_row_0/A:col10/1732762526749/Put/seqid=0 2024-11-28T02:55:27,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742177_1353 (size=48389) 2024-11-28T02:55:27,327 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/ccdd1ab569a942098be82e4b85ee2880 2024-11-28T02:55:27,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/3382e01b716c4530b7dac5da662154af is 50, key is test_row_0/B:col10/1732762526749/Put/seqid=0 2024-11-28T02:55:27,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762587336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762587337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742178_1354 (size=12151) 2024-11-28T02:55:27,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/3382e01b716c4530b7dac5da662154af 2024-11-28T02:55:27,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762587345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762587346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T02:55:27,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/9dd5538f88e74a07ba35de529610d1ca is 50, key is test_row_0/C:col10/1732762526749/Put/seqid=0 2024-11-28T02:55:27,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742179_1355 (size=12151) 2024-11-28T02:55:27,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/9dd5538f88e74a07ba35de529610d1ca 2024-11-28T02:55:27,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T02:55:27,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:27,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:27,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:27,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:27,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:27,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/ccdd1ab569a942098be82e4b85ee2880 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ccdd1ab569a942098be82e4b85ee2880 2024-11-28T02:55:27,416 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/3ae9d17168134cceaafd2069a1484c9a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/3ae9d17168134cceaafd2069a1484c9a 2024-11-28T02:55:27,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ccdd1ab569a942098be82e4b85ee2880, entries=250, sequenceid=160, filesize=47.3 K 2024-11-28T02:55:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762587406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/3382e01b716c4530b7dac5da662154af as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/3382e01b716c4530b7dac5da662154af 2024-11-28T02:55:27,427 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/A of d1bea096782fcdd0d3636fed7e5e84b2 into 3ae9d17168134cceaafd2069a1484c9a(size=30.4 K), total size for store is 116.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:27,427 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:27,427 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/A, priority=13, startTime=1732762526538; duration=0sec 2024-11-28T02:55:27,427 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:27,427 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:A 2024-11-28T02:55:27,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/3382e01b716c4530b7dac5da662154af, entries=150, sequenceid=160, filesize=11.9 K 2024-11-28T02:55:27,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/9dd5538f88e74a07ba35de529610d1ca as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/9dd5538f88e74a07ba35de529610d1ca 2024-11-28T02:55:27,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/9dd5538f88e74a07ba35de529610d1ca, entries=150, sequenceid=160, filesize=11.9 K 2024-11-28T02:55:27,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d1bea096782fcdd0d3636fed7e5e84b2 in 542ms, sequenceid=160, compaction requested=true 2024-11-28T02:55:27,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:27,436 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:27,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:27,437 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:27,437 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:27,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:27,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:27,438 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/A is initiating minor compaction (all files) 2024-11-28T02:55:27,438 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/A in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:27,438 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/3ae9d17168134cceaafd2069a1484c9a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7dcaff905d70469a8f0af29bb79a7d63, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ccdd1ab569a942098be82e4b85ee2880] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=116.5 K 2024-11-28T02:55:27,438 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:27,438 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/3ae9d17168134cceaafd2069a1484c9a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7dcaff905d70469a8f0af29bb79a7d63, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ccdd1ab569a942098be82e4b85ee2880] 2024-11-28T02:55:27,438 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:27,438 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/B is initiating minor compaction (all files) 2024-11-28T02:55:27,438 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/B in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:27,438 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/149d3925077f4a3dbb7095c6173f2f26, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/540e6caa07844367a7b9091359baad31, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/3382e01b716c4530b7dac5da662154af] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=35.7 K 2024-11-28T02:55:27,438 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ae9d17168134cceaafd2069a1484c9a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762525320 2024-11-28T02:55:27,439 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 149d3925077f4a3dbb7095c6173f2f26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762525320 2024-11-28T02:55:27,439 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dcaff905d70469a8f0af29bb79a7d63, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732762526136 2024-11-28T02:55:27,439 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 540e6caa07844367a7b9091359baad31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732762526145 2024-11-28T02:55:27,439 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccdd1ab569a942098be82e4b85ee2880, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732762526733 2024-11-28T02:55:27,440 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3382e01b716c4530b7dac5da662154af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732762526749 2024-11-28T02:55:27,455 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:27,459 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#B#compaction#301 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:27,459 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/684b5904fef94a309bf6edcb1e43503d is 50, key is test_row_0/B:col10/1732762526749/Put/seqid=0 2024-11-28T02:55:27,463 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411284cbd2704956946ed80b872abedf0b5fb_d1bea096782fcdd0d3636fed7e5e84b2 store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:27,465 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411284cbd2704956946ed80b872abedf0b5fb_d1bea096782fcdd0d3636fed7e5e84b2, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:27,465 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284cbd2704956946ed80b872abedf0b5fb_d1bea096782fcdd0d3636fed7e5e84b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:27,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742180_1356 (size=12459) 2024-11-28T02:55:27,488 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/684b5904fef94a309bf6edcb1e43503d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/684b5904fef94a309bf6edcb1e43503d 2024-11-28T02:55:27,497 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/B of d1bea096782fcdd0d3636fed7e5e84b2 into 684b5904fef94a309bf6edcb1e43503d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:27,497 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:27,497 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/B, priority=13, startTime=1732762527437; duration=0sec 2024-11-28T02:55:27,497 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:27,497 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:B 2024-11-28T02:55:27,497 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:27,498 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:27,499 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/C is initiating minor compaction (all files) 2024-11-28T02:55:27,499 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/C in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:27,499 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/a83fab5c1aa24cfda1179ce3ccd0f20b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8217ac0310e04dd3ab293ee6434f8205, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/9dd5538f88e74a07ba35de529610d1ca] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=35.7 K 2024-11-28T02:55:27,499 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting a83fab5c1aa24cfda1179ce3ccd0f20b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732762525320 2024-11-28T02:55:27,500 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8217ac0310e04dd3ab293ee6434f8205, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732762526145 2024-11-28T02:55:27,500 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dd5538f88e74a07ba35de529610d1ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732762526749 2024-11-28T02:55:27,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742181_1357 (size=4469) 2024-11-28T02:55:27,518 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#A#compaction#300 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:27,518 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/a9e7b94f1b7c4327a30be45394358fa9 is 175, key is test_row_0/A:col10/1732762526749/Put/seqid=0 2024-11-28T02:55:27,525 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:27,525 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/6ec7839e5858448fb5f5885a6d91a9d8 is 50, key is test_row_0/C:col10/1732762526749/Put/seqid=0 2024-11-28T02:55:27,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742182_1358 (size=31413) 2024-11-28T02:55:27,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742183_1359 (size=12459) 2024-11-28T02:55:27,564 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/a9e7b94f1b7c4327a30be45394358fa9 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/a9e7b94f1b7c4327a30be45394358fa9 2024-11-28T02:55:27,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T02:55:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:27,565 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T02:55:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:27,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:27,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:27,574 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/6ec7839e5858448fb5f5885a6d91a9d8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/6ec7839e5858448fb5f5885a6d91a9d8 2024-11-28T02:55:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T02:55:27,581 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/A of d1bea096782fcdd0d3636fed7e5e84b2 into a9e7b94f1b7c4327a30be45394358fa9(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:27,581 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:27,582 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/A, priority=13, startTime=1732762527436; duration=0sec 2024-11-28T02:55:27,582 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:27,582 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:A 2024-11-28T02:55:27,585 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/C of d1bea096782fcdd0d3636fed7e5e84b2 into 6ec7839e5858448fb5f5885a6d91a9d8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:27,585 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:27,585 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/C, priority=13, startTime=1732762527438; duration=0sec 2024-11-28T02:55:27,585 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:27,585 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:C 2024-11-28T02:55:27,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112875ebba8d7b22472ca2f1cf65a8e38e2f_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762527003/Put/seqid=0 2024-11-28T02:55:27,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742184_1360 (size=12304) 2024-11-28T02:55:27,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:27,624 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112875ebba8d7b22472ca2f1cf65a8e38e2f_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112875ebba8d7b22472ca2f1cf65a8e38e2f_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:27,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/b6968d12c2a04fc29a464ed4d494cd49, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:27,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/b6968d12c2a04fc29a464ed4d494cd49 is 175, key is test_row_0/A:col10/1732762527003/Put/seqid=0 2024-11-28T02:55:27,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:27,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742185_1361 (size=31105) 2024-11-28T02:55:27,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762587737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762587740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762587846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762587846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T02:55:27,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762587936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:27,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:28,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762588066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:28,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:28,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762588066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:28,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:28,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:28,084 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/b6968d12c2a04fc29a464ed4d494cd49 2024-11-28T02:55:28,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/d114cce9550b4efca98e2545fe66750d is 50, key is test_row_0/B:col10/1732762527003/Put/seqid=0 2024-11-28T02:55:28,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742186_1362 (size=12151) 2024-11-28T02:55:28,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762588366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:28,377 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in d1bea096782fcdd0d3636fed7e5e84b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:28,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:28,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762588376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:28,386 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in d1bea096782fcdd0d3636fed7e5e84b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:28,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762588386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:28,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762588388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T02:55:28,494 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/d114cce9550b4efca98e2545fe66750d 2024-11-28T02:55:28,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/fe2de5a7c87b4c09b6cb469e6b07583a is 50, key is test_row_0/C:col10/1732762527003/Put/seqid=0 2024-11-28T02:55:28,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742187_1363 (size=12151) 2024-11-28T02:55:28,538 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/fe2de5a7c87b4c09b6cb469e6b07583a 2024-11-28T02:55:28,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/b6968d12c2a04fc29a464ed4d494cd49 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/b6968d12c2a04fc29a464ed4d494cd49 2024-11-28T02:55:28,556 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/b6968d12c2a04fc29a464ed4d494cd49, entries=150, sequenceid=182, filesize=30.4 K 2024-11-28T02:55:28,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/d114cce9550b4efca98e2545fe66750d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/d114cce9550b4efca98e2545fe66750d 2024-11-28T02:55:28,566 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/d114cce9550b4efca98e2545fe66750d, entries=150, sequenceid=182, filesize=11.9 K 2024-11-28T02:55:28,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/fe2de5a7c87b4c09b6cb469e6b07583a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/fe2de5a7c87b4c09b6cb469e6b07583a 2024-11-28T02:55:28,572 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/fe2de5a7c87b4c09b6cb469e6b07583a, entries=150, sequenceid=182, filesize=11.9 K 2024-11-28T02:55:28,572 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d1bea096782fcdd0d3636fed7e5e84b2 in 1007ms, sequenceid=182, compaction requested=false 2024-11-28T02:55:28,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:28,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:28,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-28T02:55:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-28T02:55:28,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-28T02:55:28,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3160 sec 2024-11-28T02:55:28,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.3240 sec 2024-11-28T02:55:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:28,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T02:55:28,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:28,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:28,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:28,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:28,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:28,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:28,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112870da7db30fa743c09e85b054fb49c805_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762527734/Put/seqid=0 2024-11-28T02:55:28,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742188_1364 (size=14794) 2024-11-28T02:55:28,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762588971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:28,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762588981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762588985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:29,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762589096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:29,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762589096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:29,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762589097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:29,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762589307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762589307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:29,321 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:29,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762589316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:29,326 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112870da7db30fa743c09e85b054fb49c805_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112870da7db30fa743c09e85b054fb49c805_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:29,327 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/e28794dda5e84150a849fbf616e50ef0, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:29,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/e28794dda5e84150a849fbf616e50ef0 is 175, key is test_row_0/A:col10/1732762527734/Put/seqid=0 2024-11-28T02:55:29,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742189_1365 (size=39749) 2024-11-28T02:55:29,345 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/e28794dda5e84150a849fbf616e50ef0 2024-11-28T02:55:29,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/9ea64dc891dc4d41b17044320f3340ec is 50, key is test_row_0/B:col10/1732762527734/Put/seqid=0 2024-11-28T02:55:29,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742190_1366 (size=12151) 2024-11-28T02:55:29,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/9ea64dc891dc4d41b17044320f3340ec 2024-11-28T02:55:29,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/aa97d8ff86eb4234a691c7e07347de86 is 50, key is test_row_0/C:col10/1732762527734/Put/seqid=0 2024-11-28T02:55:29,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742191_1367 (size=12151) 2024-11-28T02:55:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T02:55:29,406 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:29,406 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-28T02:55:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T02:55:29,408 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:29,408 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:29,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T02:55:29,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T02:55:29,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:29,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:29,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:29,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:29,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:29,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762589626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762589626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:29,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762589636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:29,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:29,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T02:55:29,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:29,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:29,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:29,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:29,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T02:55:29,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/aa97d8ff86eb4234a691c7e07347de86 2024-11-28T02:55:29,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/e28794dda5e84150a849fbf616e50ef0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e28794dda5e84150a849fbf616e50ef0 2024-11-28T02:55:29,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e28794dda5e84150a849fbf616e50ef0, entries=200, sequenceid=198, filesize=38.8 K 2024-11-28T02:55:29,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/9ea64dc891dc4d41b17044320f3340ec as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/9ea64dc891dc4d41b17044320f3340ec 2024-11-28T02:55:29,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/9ea64dc891dc4d41b17044320f3340ec, entries=150, sequenceid=198, filesize=11.9 K 2024-11-28T02:55:29,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/aa97d8ff86eb4234a691c7e07347de86 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/aa97d8ff86eb4234a691c7e07347de86 2024-11-28T02:55:29,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/aa97d8ff86eb4234a691c7e07347de86, entries=150, sequenceid=198, filesize=11.9 K 2024-11-28T02:55:29,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d1bea096782fcdd0d3636fed7e5e84b2 in 927ms, sequenceid=198, compaction requested=true 2024-11-28T02:55:29,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:29,825 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:29,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:29,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:29,826 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:29,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:29,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:29,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:29,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:29,828 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:29,828 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/B is initiating minor compaction (all files) 2024-11-28T02:55:29,828 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/B in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:29,828 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/684b5904fef94a309bf6edcb1e43503d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/d114cce9550b4efca98e2545fe66750d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/9ea64dc891dc4d41b17044320f3340ec] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=35.9 K 2024-11-28T02:55:29,828 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:29,828 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/A is initiating minor compaction (all files) 2024-11-28T02:55:29,829 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/A in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:29,829 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/a9e7b94f1b7c4327a30be45394358fa9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/b6968d12c2a04fc29a464ed4d494cd49, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e28794dda5e84150a849fbf616e50ef0] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=99.9 K 2024-11-28T02:55:29,829 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:29,829 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/a9e7b94f1b7c4327a30be45394358fa9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/b6968d12c2a04fc29a464ed4d494cd49, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e28794dda5e84150a849fbf616e50ef0] 2024-11-28T02:55:29,829 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9e7b94f1b7c4327a30be45394358fa9, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732762526749 2024-11-28T02:55:29,829 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 684b5904fef94a309bf6edcb1e43503d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732762526749 2024-11-28T02:55:29,830 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6968d12c2a04fc29a464ed4d494cd49, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732762526984 2024-11-28T02:55:29,830 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting d114cce9550b4efca98e2545fe66750d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732762526984 2024-11-28T02:55:29,830 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e28794dda5e84150a849fbf616e50ef0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732762527718 2024-11-28T02:55:29,830 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ea64dc891dc4d41b17044320f3340ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732762527734 2024-11-28T02:55:29,845 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#B#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:29,845 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/bbd3287f1db9495c80fbfebc81b3f89c is 50, key is test_row_0/B:col10/1732762527734/Put/seqid=0 2024-11-28T02:55:29,848 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:29,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T02:55:29,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:29,866 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T02:55:29,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:29,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:29,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:29,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:29,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:29,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:29,869 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411282f3615ca5ccd48ae8b802a1bd2642619_d1bea096782fcdd0d3636fed7e5e84b2 store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:29,871 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411282f3615ca5ccd48ae8b802a1bd2642619_d1bea096782fcdd0d3636fed7e5e84b2, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:29,871 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282f3615ca5ccd48ae8b802a1bd2642619_d1bea096782fcdd0d3636fed7e5e84b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:29,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742192_1368 (size=12561) 2024-11-28T02:55:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285f7bedb99d5647129181c21d4f033512_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762528982/Put/seqid=0 2024-11-28T02:55:29,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742193_1369 (size=4469) 2024-11-28T02:55:29,946 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#A#compaction#310 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:29,946 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/252c6c6ab3ec4d2f9b92e7a100f6edaf is 175, key is test_row_0/A:col10/1732762527734/Put/seqid=0 2024-11-28T02:55:29,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742194_1370 (size=12304) 2024-11-28T02:55:29,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742195_1371 (size=31515) 2024-11-28T02:55:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T02:55:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:30,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:30,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762590168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762590169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762590182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762590286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762590286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762590295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,320 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/bbd3287f1db9495c80fbfebc81b3f89c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/bbd3287f1db9495c80fbfebc81b3f89c 2024-11-28T02:55:30,327 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/B of d1bea096782fcdd0d3636fed7e5e84b2 into bbd3287f1db9495c80fbfebc81b3f89c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:30,327 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:30,327 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/B, priority=13, startTime=1732762529826; duration=0sec 2024-11-28T02:55:30,327 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:30,327 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:B 2024-11-28T02:55:30,327 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:30,328 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:30,328 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/C is initiating minor compaction (all files) 2024-11-28T02:55:30,328 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/C in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:30,328 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/6ec7839e5858448fb5f5885a6d91a9d8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/fe2de5a7c87b4c09b6cb469e6b07583a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/aa97d8ff86eb4234a691c7e07347de86] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=35.9 K 2024-11-28T02:55:30,329 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ec7839e5858448fb5f5885a6d91a9d8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732762526749 2024-11-28T02:55:30,329 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting fe2de5a7c87b4c09b6cb469e6b07583a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732762526984 2024-11-28T02:55:30,329 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting aa97d8ff86eb4234a691c7e07347de86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732762527734 2024-11-28T02:55:30,351 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#C#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:30,353 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/8d7893afd15547b28aae12aa143dff29 is 50, key is test_row_0/C:col10/1732762527734/Put/seqid=0 2024-11-28T02:55:30,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:30,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742196_1372 (size=12561) 2024-11-28T02:55:30,361 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285f7bedb99d5647129181c21d4f033512_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285f7bedb99d5647129181c21d4f033512_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:30,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/07e1d728dbc7440dabe63b98c292ff61, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:30,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/07e1d728dbc7440dabe63b98c292ff61 is 175, key is test_row_0/A:col10/1732762528982/Put/seqid=0 2024-11-28T02:55:30,373 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/8d7893afd15547b28aae12aa143dff29 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8d7893afd15547b28aae12aa143dff29 2024-11-28T02:55:30,374 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/252c6c6ab3ec4d2f9b92e7a100f6edaf as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/252c6c6ab3ec4d2f9b92e7a100f6edaf 2024-11-28T02:55:30,379 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/A of d1bea096782fcdd0d3636fed7e5e84b2 into 252c6c6ab3ec4d2f9b92e7a100f6edaf(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:30,380 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:30,380 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/A, priority=13, startTime=1732762529825; duration=0sec 2024-11-28T02:55:30,380 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:30,380 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:A 2024-11-28T02:55:30,385 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/C of d1bea096782fcdd0d3636fed7e5e84b2 into 8d7893afd15547b28aae12aa143dff29(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:30,385 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:30,385 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/C, priority=13, startTime=1732762529826; duration=0sec 2024-11-28T02:55:30,385 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:30,385 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:C 2024-11-28T02:55:30,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742197_1373 (size=31105) 2024-11-28T02:55:30,388 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/07e1d728dbc7440dabe63b98c292ff61 2024-11-28T02:55:30,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/ee58ef245489405f9ed9abf1c691215d is 50, key is test_row_0/B:col10/1732762528982/Put/seqid=0 2024-11-28T02:55:30,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762590386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,399 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in d1bea096782fcdd0d3636fed7e5e84b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762590396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,401 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in d1bea096782fcdd0d3636fed7e5e84b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:30,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742198_1374 (size=12151) 2024-11-28T02:55:30,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762590506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762590506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762590506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T02:55:30,805 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/ee58ef245489405f9ed9abf1c691215d 2024-11-28T02:55:30,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/df1f191091c54144bcba3e52477436a3 is 50, key is test_row_0/C:col10/1732762528982/Put/seqid=0 2024-11-28T02:55:30,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762590816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762590817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:30,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762590817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:30,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:30,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:30,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742199_1375 (size=12151) 2024-11-28T02:55:30,836 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/df1f191091c54144bcba3e52477436a3 2024-11-28T02:55:30,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/07e1d728dbc7440dabe63b98c292ff61 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/07e1d728dbc7440dabe63b98c292ff61 2024-11-28T02:55:30,850 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/07e1d728dbc7440dabe63b98c292ff61, entries=150, sequenceid=216, filesize=30.4 K 2024-11-28T02:55:30,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/ee58ef245489405f9ed9abf1c691215d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/ee58ef245489405f9ed9abf1c691215d 2024-11-28T02:55:30,855 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/ee58ef245489405f9ed9abf1c691215d, entries=150, sequenceid=216, filesize=11.9 K 2024-11-28T02:55:30,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/df1f191091c54144bcba3e52477436a3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/df1f191091c54144bcba3e52477436a3 2024-11-28T02:55:30,862 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/df1f191091c54144bcba3e52477436a3, entries=150, sequenceid=216, filesize=11.9 K 2024-11-28T02:55:30,865 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d1bea096782fcdd0d3636fed7e5e84b2 in 999ms, sequenceid=216, compaction requested=false 2024-11-28T02:55:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-28T02:55:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-28T02:55:30,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-28T02:55:30,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4590 sec 2024-11-28T02:55:30,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.4630 sec 2024-11-28T02:55:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:31,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T02:55:31,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:31,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:31,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:31,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:31,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:31,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:31,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288e96eac325b9470499684b8e252347a1_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762530163/Put/seqid=0 2024-11-28T02:55:31,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742200_1376 (size=17284) 2024-11-28T02:55:31,371 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:31,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288e96eac325b9470499684b8e252347a1_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288e96eac325b9470499684b8e252347a1_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:31,377 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/ef1d037ee62c440f81f3a2138db46b13, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:31,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/ef1d037ee62c440f81f3a2138db46b13 is 175, key is test_row_0/A:col10/1732762530163/Put/seqid=0 2024-11-28T02:55:31,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742201_1377 (size=48389) 2024-11-28T02:55:31,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762591390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:31,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762591391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762591394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:31,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762591516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:31,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:31,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762591516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:31,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762591516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T02:55:31,557 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:31,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-28T02:55:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T02:55:31,562 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:31,562 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:31,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T02:55:31,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-28T02:55:31,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:31,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:31,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:31,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:31,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:31,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762591735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762591735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:31,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:31,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762591736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:31,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:31,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:31,784 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/ef1d037ee62c440f81f3a2138db46b13 2024-11-28T02:55:31,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/aa9fdf72bc4e48e6b743fbeeaa18a5a3 is 50, key is test_row_0/B:col10/1732762530163/Put/seqid=0 2024-11-28T02:55:31,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742202_1378 (size=12151) 2024-11-28T02:55:31,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/aa9fdf72bc4e48e6b743fbeeaa18a5a3 2024-11-28T02:55:31,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/e2bb255128834e55b1305628dd70fd36 is 50, key is test_row_0/C:col10/1732762530163/Put/seqid=0 2024-11-28T02:55:31,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742203_1379 (size=12151) 2024-11-28T02:55:31,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-28T02:55:31,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:31,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:31,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:31,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:31,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:31,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:31,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T02:55:32,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-28T02:55:32,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:32,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:32,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:32,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:32,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:32,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762592046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:32,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762592046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:32,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:32,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762592046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:32,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:32,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:32,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-28T02:55:32,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:32,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:32,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:32,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:32,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T02:55:32,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/e2bb255128834e55b1305628dd70fd36 2024-11-28T02:55:32,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/ef1d037ee62c440f81f3a2138db46b13 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ef1d037ee62c440f81f3a2138db46b13 2024-11-28T02:55:32,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ef1d037ee62c440f81f3a2138db46b13, entries=250, sequenceid=235, filesize=47.3 K 2024-11-28T02:55:32,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/aa9fdf72bc4e48e6b743fbeeaa18a5a3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/aa9fdf72bc4e48e6b743fbeeaa18a5a3 2024-11-28T02:55:32,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/aa9fdf72bc4e48e6b743fbeeaa18a5a3, entries=150, sequenceid=235, filesize=11.9 K 2024-11-28T02:55:32,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/e2bb255128834e55b1305628dd70fd36 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/e2bb255128834e55b1305628dd70fd36 2024-11-28T02:55:32,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/e2bb255128834e55b1305628dd70fd36, entries=150, sequenceid=235, filesize=11.9 K 2024-11-28T02:55:32,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d1bea096782fcdd0d3636fed7e5e84b2 in 881ms, sequenceid=235, compaction requested=true 2024-11-28T02:55:32,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:32,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:32,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:32,223 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:32,223 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:32,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:32,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:32,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:32,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:32,226 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:32,226 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/A is initiating minor compaction (all files) 2024-11-28T02:55:32,226 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/A in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:32,226 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/252c6c6ab3ec4d2f9b92e7a100f6edaf, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/07e1d728dbc7440dabe63b98c292ff61, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ef1d037ee62c440f81f3a2138db46b13] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=108.4 K 2024-11-28T02:55:32,226 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:32,226 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/252c6c6ab3ec4d2f9b92e7a100f6edaf, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/07e1d728dbc7440dabe63b98c292ff61, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ef1d037ee62c440f81f3a2138db46b13] 2024-11-28T02:55:32,227 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:32,227 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/B is initiating minor compaction (all files) 2024-11-28T02:55:32,227 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/B in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:32,227 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/bbd3287f1db9495c80fbfebc81b3f89c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/ee58ef245489405f9ed9abf1c691215d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/aa9fdf72bc4e48e6b743fbeeaa18a5a3] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=36.0 K 2024-11-28T02:55:32,227 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting bbd3287f1db9495c80fbfebc81b3f89c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732762527734 2024-11-28T02:55:32,227 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 252c6c6ab3ec4d2f9b92e7a100f6edaf, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732762527734 2024-11-28T02:55:32,228 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07e1d728dbc7440dabe63b98c292ff61, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732762528968 2024-11-28T02:55:32,228 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting ee58ef245489405f9ed9abf1c691215d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732762528968 2024-11-28T02:55:32,228 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef1d037ee62c440f81f3a2138db46b13, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732762530163 2024-11-28T02:55:32,229 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting aa9fdf72bc4e48e6b743fbeeaa18a5a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732762530163 2024-11-28T02:55:32,233 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:32,235 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#B#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:32,235 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112890e3332c94514638a1cecc1b159c1b81_d1bea096782fcdd0d3636fed7e5e84b2 store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:32,235 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/0d2c22ec24a348d79e2064db3e845ffb is 50, key is test_row_0/B:col10/1732762530163/Put/seqid=0 2024-11-28T02:55:32,237 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112890e3332c94514638a1cecc1b159c1b81_d1bea096782fcdd0d3636fed7e5e84b2, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:32,237 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112890e3332c94514638a1cecc1b159c1b81_d1bea096782fcdd0d3636fed7e5e84b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:32,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742204_1380 (size=4469) 2024-11-28T02:55:32,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742205_1381 (size=12663) 2024-11-28T02:55:32,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-28T02:55:32,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:32,326 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T02:55:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:32,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112882ca9aa5993c4a27b4839400d310124f_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762531385/Put/seqid=0 2024-11-28T02:55:32,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742206_1382 (size=12304) 2024-11-28T02:55:32,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:32,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:32,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762592618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:32,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:32,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762592622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:32,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762592624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:32,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:32,642 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:32,642 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/c3f3a83c28f74fa98beabc1147ad6a70 is 175, key is test_row_0/A:col10/1732762530163/Put/seqid=0 2024-11-28T02:55:32,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742207_1383 (size=31617) 2024-11-28T02:55:32,659 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/0d2c22ec24a348d79e2064db3e845ffb as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/0d2c22ec24a348d79e2064db3e845ffb 2024-11-28T02:55:32,666 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/B of d1bea096782fcdd0d3636fed7e5e84b2 into 0d2c22ec24a348d79e2064db3e845ffb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:32,666 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:32,666 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/B, priority=13, startTime=1732762532223; duration=0sec 2024-11-28T02:55:32,666 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:32,666 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:B 2024-11-28T02:55:32,666 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:32,667 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:32,667 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/C is initiating minor compaction (all files) 2024-11-28T02:55:32,667 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/C in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:32,667 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8d7893afd15547b28aae12aa143dff29, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/df1f191091c54144bcba3e52477436a3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/e2bb255128834e55b1305628dd70fd36] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=36.0 K 2024-11-28T02:55:32,667 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d7893afd15547b28aae12aa143dff29, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732762527734 2024-11-28T02:55:32,668 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting df1f191091c54144bcba3e52477436a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732762528968 2024-11-28T02:55:32,668 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting e2bb255128834e55b1305628dd70fd36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732762530163 2024-11-28T02:55:32,676 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#C#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:32,677 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/cb194a805ed5436f95a23305f9707d85 is 50, key is test_row_0/C:col10/1732762530163/Put/seqid=0 2024-11-28T02:55:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T02:55:32,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742208_1384 (size=12663) 2024-11-28T02:55:32,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:32,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762592736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762592736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:32,749 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112882ca9aa5993c4a27b4839400d310124f_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112882ca9aa5993c4a27b4839400d310124f_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/f0baf3bd022e4743b8964bf73a6584c6, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:32,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/f0baf3bd022e4743b8964bf73a6584c6 is 175, key is test_row_0/A:col10/1732762531385/Put/seqid=0 2024-11-28T02:55:32,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762592746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:32,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:32,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742209_1385 (size=31105) 2024-11-28T02:55:32,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762592956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:32,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762592956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:32,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762592966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:32,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:33,049 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/c3f3a83c28f74fa98beabc1147ad6a70 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/c3f3a83c28f74fa98beabc1147ad6a70 2024-11-28T02:55:33,052 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/A of d1bea096782fcdd0d3636fed7e5e84b2 into c3f3a83c28f74fa98beabc1147ad6a70(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:33,052 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:33,052 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/A, priority=13, startTime=1732762532223; duration=0sec 2024-11-28T02:55:33,052 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:33,052 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:A 2024-11-28T02:55:33,110 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/cb194a805ed5436f95a23305f9707d85 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/cb194a805ed5436f95a23305f9707d85 2024-11-28T02:55:33,114 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/C of d1bea096782fcdd0d3636fed7e5e84b2 into cb194a805ed5436f95a23305f9707d85(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:33,114 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:33,114 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/C, priority=13, startTime=1732762532223; duration=0sec 2024-11-28T02:55:33,114 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:33,114 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:C 2024-11-28T02:55:33,160 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/f0baf3bd022e4743b8964bf73a6584c6 2024-11-28T02:55:33,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/6dff697cf185444cb8526c2c742252fa is 50, key is test_row_0/B:col10/1732762531385/Put/seqid=0 2024-11-28T02:55:33,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742210_1386 (size=12151) 2024-11-28T02:55:33,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762593276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:33,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762593276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:33,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762593286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:33,569 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/6dff697cf185444cb8526c2c742252fa 2024-11-28T02:55:33,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/81e1709adfc340cea5c76ac86655241f is 50, key is test_row_0/C:col10/1732762531385/Put/seqid=0 2024-11-28T02:55:33,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742211_1387 (size=12151) 2024-11-28T02:55:33,585 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/81e1709adfc340cea5c76ac86655241f 2024-11-28T02:55:33,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/f0baf3bd022e4743b8964bf73a6584c6 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/f0baf3bd022e4743b8964bf73a6584c6 2024-11-28T02:55:33,592 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/f0baf3bd022e4743b8964bf73a6584c6, entries=150, sequenceid=251, filesize=30.4 K 2024-11-28T02:55:33,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/6dff697cf185444cb8526c2c742252fa as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/6dff697cf185444cb8526c2c742252fa 2024-11-28T02:55:33,596 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/6dff697cf185444cb8526c2c742252fa, entries=150, sequenceid=251, filesize=11.9 K 2024-11-28T02:55:33,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/81e1709adfc340cea5c76ac86655241f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/81e1709adfc340cea5c76ac86655241f 2024-11-28T02:55:33,601 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/81e1709adfc340cea5c76ac86655241f, entries=150, sequenceid=251, filesize=11.9 K 2024-11-28T02:55:33,602 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d1bea096782fcdd0d3636fed7e5e84b2 in 1275ms, sequenceid=251, compaction requested=false 2024-11-28T02:55:33,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:33,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:33,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-28T02:55:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-28T02:55:33,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-28T02:55:33,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0430 sec 2024-11-28T02:55:33,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.0490 sec 2024-11-28T02:55:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T02:55:33,706 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:33,706 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-28T02:55:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T02:55:33,708 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:33,708 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:33,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:33,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T02:55:33,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:33,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:33,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:33,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:33,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:33,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:33,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d230a2748ebe402c94cee1c29abdb178_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762532622/Put/seqid=0 2024-11-28T02:55:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T02:55:33,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742212_1388 (size=14994) 2024-11-28T02:55:33,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:33,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762593842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762593843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:33,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762593847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:33,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T02:55:33,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:33,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:33,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:33,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:33,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:33,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762593956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:33,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:33,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762593956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:33,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:33,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762593956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:33,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:34,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T02:55:34,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:34,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T02:55:34,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T02:55:34,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:34,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762594177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:34,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762594177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:34,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762594177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:34,224 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:34,227 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d230a2748ebe402c94cee1c29abdb178_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d230a2748ebe402c94cee1c29abdb178_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:34,228 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/e574ab12cf194a5eba5d0cac2310a8bd, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:34,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/e574ab12cf194a5eba5d0cac2310a8bd is 175, key is test_row_0/A:col10/1732762532622/Put/seqid=0 2024-11-28T02:55:34,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742213_1389 (size=39949) 2024-11-28T02:55:34,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T02:55:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T02:55:34,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45926 deadline: 1732762594416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:34,418 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in d1bea096782fcdd0d3636fed7e5e84b2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:34,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:34,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45958 deadline: 1732762594426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:34,430 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in d1bea096782fcdd0d3636fed7e5e84b2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:34,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T02:55:34,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:34,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:34,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762594496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:34,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:34,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762594496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:34,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:34,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762594496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:34,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T02:55:34,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:34,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:34,632 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/e574ab12cf194a5eba5d0cac2310a8bd 2024-11-28T02:55:34,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/8faeec23e63e4405b51a27112b155953 is 50, key is test_row_0/B:col10/1732762532622/Put/seqid=0 2024-11-28T02:55:34,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742214_1390 (size=12301) 2024-11-28T02:55:34,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/8faeec23e63e4405b51a27112b155953 2024-11-28T02:55:34,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/2dbd77cc46704a65bc94e912740156f9 is 50, key is test_row_0/C:col10/1732762532622/Put/seqid=0 2024-11-28T02:55:34,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742215_1391 (size=12301) 2024-11-28T02:55:34,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/2dbd77cc46704a65bc94e912740156f9 2024-11-28T02:55:34,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/e574ab12cf194a5eba5d0cac2310a8bd as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e574ab12cf194a5eba5d0cac2310a8bd 2024-11-28T02:55:34,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e574ab12cf194a5eba5d0cac2310a8bd, entries=200, sequenceid=271, filesize=39.0 K 2024-11-28T02:55:34,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/8faeec23e63e4405b51a27112b155953 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/8faeec23e63e4405b51a27112b155953 2024-11-28T02:55:34,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/8faeec23e63e4405b51a27112b155953, entries=150, sequenceid=271, filesize=12.0 K 2024-11-28T02:55:34,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/2dbd77cc46704a65bc94e912740156f9 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/2dbd77cc46704a65bc94e912740156f9 2024-11-28T02:55:34,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/2dbd77cc46704a65bc94e912740156f9, entries=150, sequenceid=271, filesize=12.0 K 2024-11-28T02:55:34,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d1bea096782fcdd0d3636fed7e5e84b2 in 896ms, sequenceid=271, compaction requested=true 2024-11-28T02:55:34,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:34,704 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:34,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:34,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:34,705 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:34,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:34,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:34,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:34,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:34,706 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:34,706 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:34,706 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/A is initiating minor compaction (all files) 2024-11-28T02:55:34,706 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/B is initiating minor compaction (all files) 2024-11-28T02:55:34,706 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/B in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,706 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/A in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,706 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/0d2c22ec24a348d79e2064db3e845ffb, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/6dff697cf185444cb8526c2c742252fa, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/8faeec23e63e4405b51a27112b155953] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=36.2 K 2024-11-28T02:55:34,706 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/c3f3a83c28f74fa98beabc1147ad6a70, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/f0baf3bd022e4743b8964bf73a6584c6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e574ab12cf194a5eba5d0cac2310a8bd] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=100.3 K 2024-11-28T02:55:34,706 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,706 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/c3f3a83c28f74fa98beabc1147ad6a70, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/f0baf3bd022e4743b8964bf73a6584c6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e574ab12cf194a5eba5d0cac2310a8bd] 2024-11-28T02:55:34,707 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d2c22ec24a348d79e2064db3e845ffb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732762530163 2024-11-28T02:55:34,707 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3f3a83c28f74fa98beabc1147ad6a70, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732762530163 2024-11-28T02:55:34,707 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dff697cf185444cb8526c2c742252fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732762531385 2024-11-28T02:55:34,707 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0baf3bd022e4743b8964bf73a6584c6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732762531385 2024-11-28T02:55:34,708 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8faeec23e63e4405b51a27112b155953, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732762532621 2024-11-28T02:55:34,708 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e574ab12cf194a5eba5d0cac2310a8bd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732762532609 2024-11-28T02:55:34,714 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:34,715 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/b63234a1f0ea470697e77bd24024c501 is 50, key is test_row_0/B:col10/1732762532622/Put/seqid=0 2024-11-28T02:55:34,715 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:34,719 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128be4aeefd41874c80af3fecbf7f70e2e4_d1bea096782fcdd0d3636fed7e5e84b2 store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:34,721 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128be4aeefd41874c80af3fecbf7f70e2e4_d1bea096782fcdd0d3636fed7e5e84b2, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:34,721 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128be4aeefd41874c80af3fecbf7f70e2e4_d1bea096782fcdd0d3636fed7e5e84b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:34,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742216_1392 (size=12915) 2024-11-28T02:55:34,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742217_1393 (size=4469) 2024-11-28T02:55:34,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T02:55:34,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:34,779 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T02:55:34,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:34,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:34,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:34,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:34,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:34,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:34,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c7c382180131466fa741aeb85baf38cc_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762533841/Put/seqid=0 2024-11-28T02:55:34,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742218_1394 (size=12454) 2024-11-28T02:55:34,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:34,820 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c7c382180131466fa741aeb85baf38cc_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c7c382180131466fa741aeb85baf38cc_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:34,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/9750ca32664d4c989a216a1b7b86f0a3, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/9750ca32664d4c989a216a1b7b86f0a3 is 175, key is test_row_0/A:col10/1732762533841/Put/seqid=0 2024-11-28T02:55:34,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742219_1395 (size=31255) 2024-11-28T02:55:34,834 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/9750ca32664d4c989a216a1b7b86f0a3 2024-11-28T02:55:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T02:55:34,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/9fbafd4fade145c881e3b2d3aa8457bd is 50, key is test_row_0/B:col10/1732762533841/Put/seqid=0 2024-11-28T02:55:34,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742220_1396 (size=12301) 2024-11-28T02:55:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:35,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:35,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762595072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762595082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762595084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,145 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#A#compaction#328 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:35,146 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/40bd0f07ded5449291d438aea3e0658c is 175, key is test_row_0/A:col10/1732762532622/Put/seqid=0 2024-11-28T02:55:35,155 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/b63234a1f0ea470697e77bd24024c501 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/b63234a1f0ea470697e77bd24024c501 2024-11-28T02:55:35,168 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/B of d1bea096782fcdd0d3636fed7e5e84b2 into b63234a1f0ea470697e77bd24024c501(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:35,168 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:35,168 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/B, priority=13, startTime=1732762534705; duration=0sec 2024-11-28T02:55:35,168 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:35,168 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:B 2024-11-28T02:55:35,168 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:35,170 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:35,170 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/C is initiating minor compaction (all files) 2024-11-28T02:55:35,170 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/C in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:35,171 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/cb194a805ed5436f95a23305f9707d85, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/81e1709adfc340cea5c76ac86655241f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/2dbd77cc46704a65bc94e912740156f9] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=36.2 K 2024-11-28T02:55:35,171 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting cb194a805ed5436f95a23305f9707d85, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732762530163 2024-11-28T02:55:35,171 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e1709adfc340cea5c76ac86655241f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732762531385 2024-11-28T02:55:35,171 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dbd77cc46704a65bc94e912740156f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732762532621 2024-11-28T02:55:35,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742221_1397 (size=31869) 2024-11-28T02:55:35,184 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/40bd0f07ded5449291d438aea3e0658c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/40bd0f07ded5449291d438aea3e0658c 2024-11-28T02:55:35,191 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/A of d1bea096782fcdd0d3636fed7e5e84b2 into 40bd0f07ded5449291d438aea3e0658c(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:35,191 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:35,191 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/A, priority=13, startTime=1732762534704; duration=0sec 2024-11-28T02:55:35,191 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:35,191 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:A 2024-11-28T02:55:35,191 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#C#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:35,192 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/2f889ec855b24dcea9206403e4219289 is 50, key is test_row_0/C:col10/1732762532622/Put/seqid=0 2024-11-28T02:55:35,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762595196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762595196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762595207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742222_1398 (size=12915) 2024-11-28T02:55:35,220 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/2f889ec855b24dcea9206403e4219289 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/2f889ec855b24dcea9206403e4219289 2024-11-28T02:55:35,225 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/C of d1bea096782fcdd0d3636fed7e5e84b2 into 2f889ec855b24dcea9206403e4219289(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:35,225 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:35,225 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/C, priority=13, startTime=1732762534705; duration=0sec 2024-11-28T02:55:35,225 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:35,226 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:C 2024-11-28T02:55:35,252 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/9fbafd4fade145c881e3b2d3aa8457bd 2024-11-28T02:55:35,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/0116af6c17904137b23175a696132c8d is 50, key is test_row_0/C:col10/1732762533841/Put/seqid=0 2024-11-28T02:55:35,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742223_1399 (size=12301) 2024-11-28T02:55:35,293 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/0116af6c17904137b23175a696132c8d 2024-11-28T02:55:35,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/9750ca32664d4c989a216a1b7b86f0a3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/9750ca32664d4c989a216a1b7b86f0a3 2024-11-28T02:55:35,302 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/9750ca32664d4c989a216a1b7b86f0a3, entries=150, sequenceid=286, filesize=30.5 K 2024-11-28T02:55:35,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/9fbafd4fade145c881e3b2d3aa8457bd as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/9fbafd4fade145c881e3b2d3aa8457bd 2024-11-28T02:55:35,307 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/9fbafd4fade145c881e3b2d3aa8457bd, entries=150, sequenceid=286, filesize=12.0 K 2024-11-28T02:55:35,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/0116af6c17904137b23175a696132c8d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/0116af6c17904137b23175a696132c8d 2024-11-28T02:55:35,332 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/0116af6c17904137b23175a696132c8d, entries=150, sequenceid=286, filesize=12.0 K 2024-11-28T02:55:35,333 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d1bea096782fcdd0d3636fed7e5e84b2 in 554ms, sequenceid=286, compaction requested=false 2024-11-28T02:55:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-28T02:55:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-28T02:55:35,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-28T02:55:35,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6320 sec 2024-11-28T02:55:35,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.6370 sec 2024-11-28T02:55:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:35,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T02:55:35,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:35,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:35,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:35,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:35,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:35,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:35,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112803227fafa1fe46ffaac4b1f62dc6ecc3_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762535421/Put/seqid=0 2024-11-28T02:55:35,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742224_1400 (size=14994) 2024-11-28T02:55:35,438 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:35,441 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112803227fafa1fe46ffaac4b1f62dc6ecc3_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112803227fafa1fe46ffaac4b1f62dc6ecc3_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:35,441 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/4c3c56557c654cabac130384e0f23917, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:35,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/4c3c56557c654cabac130384e0f23917 is 175, key is test_row_0/A:col10/1732762535421/Put/seqid=0 2024-11-28T02:55:35,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742225_1401 (size=39949) 2024-11-28T02:55:35,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762595451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762595462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762595462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762595566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762595576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762595576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762595776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762595796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762595796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:35,845 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/4c3c56557c654cabac130384e0f23917 2024-11-28T02:55:35,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/75910f943f724725a4da957f8d82caf7 is 50, key is test_row_0/B:col10/1732762535421/Put/seqid=0 2024-11-28T02:55:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T02:55:35,856 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:35,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-28T02:55:35,858 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:35,859 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:35,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T02:55:35,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742226_1402 (size=12301) 2024-11-28T02:55:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T02:55:36,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T02:55:36,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:36,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762596096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:36,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762596106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:36,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:36,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:36,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:36,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762596116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:36,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T02:55:36,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:36,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T02:55:36,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/75910f943f724725a4da957f8d82caf7 2024-11-28T02:55:36,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/1c3abbf192d14bf09f19bd24d064909e is 50, key is test_row_0/C:col10/1732762535421/Put/seqid=0 2024-11-28T02:55:36,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742227_1403 (size=12301) 2024-11-28T02:55:36,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T02:55:36,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:36,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T02:55:36,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:36,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T02:55:36,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762596606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:36,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T02:55:36,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:36,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762596626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:36,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762596636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:36,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/1c3abbf192d14bf09f19bd24d064909e 2024-11-28T02:55:36,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/4c3c56557c654cabac130384e0f23917 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/4c3c56557c654cabac130384e0f23917 2024-11-28T02:55:36,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/4c3c56557c654cabac130384e0f23917, entries=200, sequenceid=307, filesize=39.0 K 2024-11-28T02:55:36,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/75910f943f724725a4da957f8d82caf7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/75910f943f724725a4da957f8d82caf7 2024-11-28T02:55:36,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/75910f943f724725a4da957f8d82caf7, entries=150, sequenceid=307, filesize=12.0 K 2024-11-28T02:55:36,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/1c3abbf192d14bf09f19bd24d064909e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/1c3abbf192d14bf09f19bd24d064909e 2024-11-28T02:55:36,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/1c3abbf192d14bf09f19bd24d064909e, entries=150, sequenceid=307, filesize=12.0 K 2024-11-28T02:55:36,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d1bea096782fcdd0d3636fed7e5e84b2 in 1286ms, sequenceid=307, compaction requested=true 2024-11-28T02:55:36,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:36,708 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:36,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:36,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:36,709 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:36,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:36,709 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:36,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:36,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:36,709 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/A is initiating minor compaction (all files) 2024-11-28T02:55:36,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:36,709 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/A in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,709 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/40bd0f07ded5449291d438aea3e0658c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/9750ca32664d4c989a216a1b7b86f0a3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/4c3c56557c654cabac130384e0f23917] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=100.7 K 2024-11-28T02:55:36,709 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,709 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/40bd0f07ded5449291d438aea3e0658c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/9750ca32664d4c989a216a1b7b86f0a3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/4c3c56557c654cabac130384e0f23917] 2024-11-28T02:55:36,710 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40bd0f07ded5449291d438aea3e0658c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732762532621 2024-11-28T02:55:36,710 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:36,710 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/B is initiating minor compaction (all files) 2024-11-28T02:55:36,710 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/B in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,710 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/b63234a1f0ea470697e77bd24024c501, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/9fbafd4fade145c881e3b2d3aa8457bd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/75910f943f724725a4da957f8d82caf7] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=36.6 K 2024-11-28T02:55:36,710 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9750ca32664d4c989a216a1b7b86f0a3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732762533841 2024-11-28T02:55:36,710 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting b63234a1f0ea470697e77bd24024c501, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732762532621 2024-11-28T02:55:36,711 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fbafd4fade145c881e3b2d3aa8457bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732762533841 2024-11-28T02:55:36,711 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c3c56557c654cabac130384e0f23917, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732762535062 2024-11-28T02:55:36,711 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 75910f943f724725a4da957f8d82caf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732762535080 2024-11-28T02:55:36,722 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:36,723 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:36,723 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128c5de068e9098478081ac083aa0bea5e2_d1bea096782fcdd0d3636fed7e5e84b2 store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:36,724 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/896c541fa1bd41f5b355453f89ac096c is 50, key is test_row_0/B:col10/1732762535421/Put/seqid=0 2024-11-28T02:55:36,726 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128c5de068e9098478081ac083aa0bea5e2_d1bea096782fcdd0d3636fed7e5e84b2, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:36,726 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c5de068e9098478081ac083aa0bea5e2_d1bea096782fcdd0d3636fed7e5e84b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:36,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742229_1405 (size=13017) 2024-11-28T02:55:36,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742228_1404 (size=4469) 2024-11-28T02:55:36,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T02:55:36,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:36,776 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T02:55:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282096c224cef84c92aeb1efacb6ffea40_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762535449/Put/seqid=0 2024-11-28T02:55:36,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742230_1406 (size=12454) 2024-11-28T02:55:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T02:55:37,142 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#A#compaction#337 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:37,142 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/896c541fa1bd41f5b355453f89ac096c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/896c541fa1bd41f5b355453f89ac096c 2024-11-28T02:55:37,142 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/df9f496bc58140649191f7ff56ae0e8a is 175, key is test_row_0/A:col10/1732762535421/Put/seqid=0 2024-11-28T02:55:37,148 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/B of d1bea096782fcdd0d3636fed7e5e84b2 into 896c541fa1bd41f5b355453f89ac096c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:37,148 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:37,148 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/B, priority=13, startTime=1732762536708; duration=0sec 2024-11-28T02:55:37,148 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:37,149 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:B 2024-11-28T02:55:37,149 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:37,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742231_1407 (size=31971) 2024-11-28T02:55:37,152 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:37,152 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/C is initiating minor compaction (all files) 2024-11-28T02:55:37,152 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/C in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:37,152 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/2f889ec855b24dcea9206403e4219289, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/0116af6c17904137b23175a696132c8d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/1c3abbf192d14bf09f19bd24d064909e] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=36.6 K 2024-11-28T02:55:37,153 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f889ec855b24dcea9206403e4219289, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732762532621 2024-11-28T02:55:37,153 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0116af6c17904137b23175a696132c8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732762533841 2024-11-28T02:55:37,154 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c3abbf192d14bf09f19bd24d064909e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732762535080 2024-11-28T02:55:37,163 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#C#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:37,163 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/21d654593fb244dbb9fd6cd304e6c002 is 50, key is test_row_0/C:col10/1732762535421/Put/seqid=0 2024-11-28T02:55:37,166 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/df9f496bc58140649191f7ff56ae0e8a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/df9f496bc58140649191f7ff56ae0e8a 2024-11-28T02:55:37,172 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/A of d1bea096782fcdd0d3636fed7e5e84b2 into df9f496bc58140649191f7ff56ae0e8a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:37,172 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:37,172 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/A, priority=13, startTime=1732762536708; duration=0sec 2024-11-28T02:55:37,172 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:37,172 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:A 2024-11-28T02:55:37,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:37,190 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282096c224cef84c92aeb1efacb6ffea40_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282096c224cef84c92aeb1efacb6ffea40_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:37,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/02ca6971618d497480e3f41f332e7a21, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:37,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/02ca6971618d497480e3f41f332e7a21 is 175, key is test_row_0/A:col10/1732762535449/Put/seqid=0 2024-11-28T02:55:37,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742232_1408 (size=13017) 2024-11-28T02:55:37,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742233_1409 (size=31255) 2024-11-28T02:55:37,203 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/02ca6971618d497480e3f41f332e7a21 2024-11-28T02:55:37,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/41b149fde1ac4621a8dbcb53458559c8 is 50, key is test_row_0/B:col10/1732762535449/Put/seqid=0 2024-11-28T02:55:37,218 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/21d654593fb244dbb9fd6cd304e6c002 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/21d654593fb244dbb9fd6cd304e6c002 2024-11-28T02:55:37,224 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/C of d1bea096782fcdd0d3636fed7e5e84b2 into 21d654593fb244dbb9fd6cd304e6c002(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:37,224 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:37,224 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/C, priority=13, startTime=1732762536709; duration=0sec 2024-11-28T02:55:37,224 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:37,224 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:C 2024-11-28T02:55:37,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742234_1410 (size=12301) 2024-11-28T02:55:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:37,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:37,649 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/41b149fde1ac4621a8dbcb53458559c8 2024-11-28T02:55:37,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/f0058726fa384c568ddb8ccb0303a105 is 50, key is test_row_0/C:col10/1732762535449/Put/seqid=0 2024-11-28T02:55:37,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742235_1411 (size=12301) 2024-11-28T02:55:37,665 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/f0058726fa384c568ddb8ccb0303a105 2024-11-28T02:55:37,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/02ca6971618d497480e3f41f332e7a21 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/02ca6971618d497480e3f41f332e7a21 2024-11-28T02:55:37,678 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/02ca6971618d497480e3f41f332e7a21, entries=150, sequenceid=321, filesize=30.5 K 2024-11-28T02:55:37,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/41b149fde1ac4621a8dbcb53458559c8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/41b149fde1ac4621a8dbcb53458559c8 2024-11-28T02:55:37,693 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/41b149fde1ac4621a8dbcb53458559c8, entries=150, sequenceid=321, filesize=12.0 K 2024-11-28T02:55:37,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/f0058726fa384c568ddb8ccb0303a105 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/f0058726fa384c568ddb8ccb0303a105 2024-11-28T02:55:37,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762597693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:37,698 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/f0058726fa384c568ddb8ccb0303a105, entries=150, sequenceid=321, filesize=12.0 K 2024-11-28T02:55:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:37,698 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d1bea096782fcdd0d3636fed7e5e84b2 in 922ms, sequenceid=321, compaction requested=false 2024-11-28T02:55:37,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:37,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:37,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-28T02:55:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-28T02:55:37,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-28T02:55:37,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8410 sec 2024-11-28T02:55:37,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8460 sec 2024-11-28T02:55:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:37,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T02:55:37,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:37,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:37,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:37,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:37,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:37,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:37,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288787e0422ebc4b89858ecdccdd278a4b_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762537667/Put/seqid=0 2024-11-28T02:55:37,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742236_1412 (size=14994) 2024-11-28T02:55:37,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762597771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:37,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762597789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:37,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762597806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:37,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762597896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:37,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762597896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:37,997 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T02:55:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T02:55:38,006 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:38,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-28T02:55:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T02:55:38,008 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:38,009 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:38,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:38,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762598016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:38,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762598106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762598106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T02:55:38,126 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:38,130 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288787e0422ebc4b89858ecdccdd278a4b_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288787e0422ebc4b89858ecdccdd278a4b_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:38,131 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/dbaf969f2e3044ea976e6bfde3565cc8, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:38,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/dbaf969f2e3044ea976e6bfde3565cc8 is 175, key is test_row_0/A:col10/1732762537667/Put/seqid=0 2024-11-28T02:55:38,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742237_1413 (size=39949) 2024-11-28T02:55:38,141 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/dbaf969f2e3044ea976e6bfde3565cc8 2024-11-28T02:55:38,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/4623f529326747e69a9df56873e5617e is 50, key is test_row_0/B:col10/1732762537667/Put/seqid=0 2024-11-28T02:55:38,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742238_1414 (size=12301) 2024-11-28T02:55:38,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T02:55:38,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:38,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:38,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:38,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T02:55:38,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:38,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:38,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:38,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T02:55:38,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762598326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:38,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762598416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:38,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:38,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762598416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:38,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:38,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T02:55:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:38,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/4623f529326747e69a9df56873e5617e 2024-11-28T02:55:38,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/b2f164adbf3c4a0caf46daafdff1988e is 50, key is test_row_0/C:col10/1732762537667/Put/seqid=0 2024-11-28T02:55:38,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742239_1415 (size=12301) 2024-11-28T02:55:38,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T02:55:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:38,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T02:55:38,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T02:55:38,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:38,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:38,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:38,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762598836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:38,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:38,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762598926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:38,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T02:55:38,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:38,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:38,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:38,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:38,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762598936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:39,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/b2f164adbf3c4a0caf46daafdff1988e 2024-11-28T02:55:39,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/dbaf969f2e3044ea976e6bfde3565cc8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbaf969f2e3044ea976e6bfde3565cc8 2024-11-28T02:55:39,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbaf969f2e3044ea976e6bfde3565cc8, entries=200, sequenceid=343, filesize=39.0 K 2024-11-28T02:55:39,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/4623f529326747e69a9df56873e5617e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/4623f529326747e69a9df56873e5617e 2024-11-28T02:55:39,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/4623f529326747e69a9df56873e5617e, entries=150, sequenceid=343, filesize=12.0 K 2024-11-28T02:55:39,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/b2f164adbf3c4a0caf46daafdff1988e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/b2f164adbf3c4a0caf46daafdff1988e 2024-11-28T02:55:39,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/b2f164adbf3c4a0caf46daafdff1988e, entries=150, sequenceid=343, filesize=12.0 K 2024-11-28T02:55:39,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d1bea096782fcdd0d3636fed7e5e84b2 in 1339ms, sequenceid=343, compaction requested=true 2024-11-28T02:55:39,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:39,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:39,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:39,045 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:39,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:39,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:39,045 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:39,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:39,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:39,046 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:39,046 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:39,046 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/A is initiating minor compaction (all files) 2024-11-28T02:55:39,046 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/B is initiating minor compaction (all files) 2024-11-28T02:55:39,046 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/B in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:39,046 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/A in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:39,046 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/896c541fa1bd41f5b355453f89ac096c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/41b149fde1ac4621a8dbcb53458559c8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/4623f529326747e69a9df56873e5617e] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=36.7 K 2024-11-28T02:55:39,046 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/df9f496bc58140649191f7ff56ae0e8a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/02ca6971618d497480e3f41f332e7a21, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbaf969f2e3044ea976e6bfde3565cc8] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=100.8 K 2024-11-28T02:55:39,046 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:39,046 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/df9f496bc58140649191f7ff56ae0e8a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/02ca6971618d497480e3f41f332e7a21, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbaf969f2e3044ea976e6bfde3565cc8] 2024-11-28T02:55:39,046 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting df9f496bc58140649191f7ff56ae0e8a, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732762535080 2024-11-28T02:55:39,046 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 896c541fa1bd41f5b355453f89ac096c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732762535080 2024-11-28T02:55:39,047 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 41b149fde1ac4621a8dbcb53458559c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732762535449 2024-11-28T02:55:39,047 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02ca6971618d497480e3f41f332e7a21, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732762535449 2024-11-28T02:55:39,047 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4623f529326747e69a9df56873e5617e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732762537667 2024-11-28T02:55:39,047 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbaf969f2e3044ea976e6bfde3565cc8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732762537667 2024-11-28T02:55:39,056 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:39,057 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:39,057 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/5c7278721b654b6aa0ddc27c2d9dc4fc is 50, key is test_row_0/B:col10/1732762537667/Put/seqid=0 2024-11-28T02:55:39,063 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128748345e0b1f44a459d9e0b35a9c9751c_d1bea096782fcdd0d3636fed7e5e84b2 store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:39,065 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128748345e0b1f44a459d9e0b35a9c9751c_d1bea096782fcdd0d3636fed7e5e84b2, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:39,065 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128748345e0b1f44a459d9e0b35a9c9751c_d1bea096782fcdd0d3636fed7e5e84b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:39,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742240_1416 (size=13119) 2024-11-28T02:55:39,076 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/5c7278721b654b6aa0ddc27c2d9dc4fc as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/5c7278721b654b6aa0ddc27c2d9dc4fc 2024-11-28T02:55:39,084 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/B of d1bea096782fcdd0d3636fed7e5e84b2 into 5c7278721b654b6aa0ddc27c2d9dc4fc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:39,084 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:39,084 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/B, priority=13, startTime=1732762539045; duration=0sec 2024-11-28T02:55:39,084 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:39,084 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:B 2024-11-28T02:55:39,084 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:39,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742241_1417 (size=4469) 2024-11-28T02:55:39,086 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:39,086 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/C is initiating minor compaction (all files) 2024-11-28T02:55:39,086 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/C in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:39,087 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/21d654593fb244dbb9fd6cd304e6c002, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/f0058726fa384c568ddb8ccb0303a105, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/b2f164adbf3c4a0caf46daafdff1988e] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=36.7 K 2024-11-28T02:55:39,087 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d654593fb244dbb9fd6cd304e6c002, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732762535080 2024-11-28T02:55:39,087 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting f0058726fa384c568ddb8ccb0303a105, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732762535449 2024-11-28T02:55:39,088 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f164adbf3c4a0caf46daafdff1988e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732762537667 2024-11-28T02:55:39,089 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#A#compaction#346 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:39,090 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/7b531599a48248bb9b40430eb7e65137 is 175, key is test_row_0/A:col10/1732762537667/Put/seqid=0 2024-11-28T02:55:39,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-28T02:55:39,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:39,095 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T02:55:39,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:39,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:39,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:39,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:39,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:39,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:39,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742242_1418 (size=32073) 2024-11-28T02:55:39,104 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#C#compaction#347 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:39,104 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/a485bfae247c43059207ef72e66de090 is 50, key is test_row_0/C:col10/1732762537667/Put/seqid=0 2024-11-28T02:55:39,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742243_1419 (size=13119) 2024-11-28T02:55:39,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280dc645ef66e84f159dff53f99ce90a8a_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762537750/Put/seqid=0 2024-11-28T02:55:39,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742244_1420 (size=12454) 2024-11-28T02:55:39,119 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/a485bfae247c43059207ef72e66de090 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/a485bfae247c43059207ef72e66de090 2024-11-28T02:55:39,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,123 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280dc645ef66e84f159dff53f99ce90a8a_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280dc645ef66e84f159dff53f99ce90a8a_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:39,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/071fb77facc348f69406630c5152e5e2, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:39,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/071fb77facc348f69406630c5152e5e2 is 175, key is test_row_0/A:col10/1732762537750/Put/seqid=0 2024-11-28T02:55:39,127 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/C of d1bea096782fcdd0d3636fed7e5e84b2 into a485bfae247c43059207ef72e66de090(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:39,127 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:39,127 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/C, priority=13, startTime=1732762539045; duration=0sec 2024-11-28T02:55:39,127 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:39,127 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:C 2024-11-28T02:55:39,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742245_1421 (size=31255) 2024-11-28T02:55:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T02:55:39,148 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/071fb77facc348f69406630c5152e5e2 2024-11-28T02:55:39,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/f864282431b34629b3babb966da3faa8 is 50, key is test_row_0/B:col10/1732762537750/Put/seqid=0 2024-11-28T02:55:39,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742246_1422 (size=12301) 2024-11-28T02:55:39,196 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/f864282431b34629b3babb966da3faa8 2024-11-28T02:55:39,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/f4f2ccdd12b54464870b2873379ce87e is 50, key is test_row_0/C:col10/1732762537750/Put/seqid=0 2024-11-28T02:55:39,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742247_1423 (size=12301) 2024-11-28T02:55:39,236 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/f4f2ccdd12b54464870b2873379ce87e 2024-11-28T02:55:39,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/071fb77facc348f69406630c5152e5e2 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/071fb77facc348f69406630c5152e5e2 2024-11-28T02:55:39,249 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/071fb77facc348f69406630c5152e5e2, entries=150, sequenceid=357, filesize=30.5 K 2024-11-28T02:55:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/f864282431b34629b3babb966da3faa8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/f864282431b34629b3babb966da3faa8 2024-11-28T02:55:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,260 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/f864282431b34629b3babb966da3faa8, entries=150, sequenceid=357, filesize=12.0 K 2024-11-28T02:55:39,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/f4f2ccdd12b54464870b2873379ce87e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/f4f2ccdd12b54464870b2873379ce87e 2024-11-28T02:55:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,268 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/f4f2ccdd12b54464870b2873379ce87e, entries=150, sequenceid=357, filesize=12.0 K 2024-11-28T02:55:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,270 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for d1bea096782fcdd0d3636fed7e5e84b2 in 174ms, sequenceid=357, compaction requested=false 2024-11-28T02:55:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:39,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:39,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-28T02:55:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-28T02:55:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-28T02:55:39,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2630 sec 2024-11-28T02:55:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.2690 sec 2024-11-28T02:55:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,506 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/7b531599a48248bb9b40430eb7e65137 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7b531599a48248bb9b40430eb7e65137 2024-11-28T02:55:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,511 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/A of d1bea096782fcdd0d3636fed7e5e84b2 into 7b531599a48248bb9b40430eb7e65137(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:39,511 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:39,511 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/A, priority=13, startTime=1732762539045; duration=0sec 2024-11-28T02:55:39,511 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:39,511 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:A 2024-11-28T02:55:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T02:55:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:40,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:40,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:40,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:40,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:40,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285c4d54283c344ef5b05fc06f4eba129f_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762540035/Put/seqid=0 2024-11-28T02:55:40,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742248_1424 (size=17534) 2024-11-28T02:55:40,152 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T02:55:40,156 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:40,156 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-28T02:55:40,158 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T02:55:40,159 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:40,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:40,168 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285c4d54283c344ef5b05fc06f4eba129f_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285c4d54283c344ef5b05fc06f4eba129f_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:40,172 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/edd251a0eb2a4023b38ebe7538343ffc, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:40,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/edd251a0eb2a4023b38ebe7538343ffc is 175, key is test_row_0/A:col10/1732762540035/Put/seqid=0 2024-11-28T02:55:40,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762540161, max timestamp=1732762540195Num uniques 859; 2024-11-28T02:55:40,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:40,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762540161, max timestamp=1732762540195Num uniques 859; 2024-11-28T02:55:40,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:40,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762540161, max timestamp=1732762540195Num uniques 859; 2024-11-28T02:55:40,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:40,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742249_1425 (size=48639) 2024-11-28T02:55:40,226 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/edd251a0eb2a4023b38ebe7538343ffc 2024-11-28T02:55:40,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/eeab6c272d41483492f99d8573b22951 is 50, key is test_row_0/B:col10/1732762540035/Put/seqid=0 2024-11-28T02:55:40,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742250_1426 (size=12301) 2024-11-28T02:55:40,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/eeab6c272d41483492f99d8573b22951 2024-11-28T02:55:40,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/aa56e23c3a61409e99938dff4b05c102 is 50, key is test_row_0/C:col10/1732762540035/Put/seqid=0 2024-11-28T02:55:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T02:55:40,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762600266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:40,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742251_1427 (size=12301) 2024-11-28T02:55:40,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/aa56e23c3a61409e99938dff4b05c102 2024-11-28T02:55:40,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:40,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762600268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762600275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:40,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/edd251a0eb2a4023b38ebe7538343ffc as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/edd251a0eb2a4023b38ebe7538343ffc 2024-11-28T02:55:40,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/edd251a0eb2a4023b38ebe7538343ffc, entries=250, sequenceid=369, filesize=47.5 K 2024-11-28T02:55:40,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/eeab6c272d41483492f99d8573b22951 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/eeab6c272d41483492f99d8573b22951 2024-11-28T02:55:40,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T02:55:40,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:40,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/eeab6c272d41483492f99d8573b22951, entries=150, sequenceid=369, filesize=12.0 K 2024-11-28T02:55:40,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/aa56e23c3a61409e99938dff4b05c102 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/aa56e23c3a61409e99938dff4b05c102 2024-11-28T02:55:40,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/aa56e23c3a61409e99938dff4b05c102, entries=150, sequenceid=369, filesize=12.0 K 2024-11-28T02:55:40,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d1bea096782fcdd0d3636fed7e5e84b2 in 232ms, sequenceid=369, compaction requested=true 2024-11-28T02:55:40,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:40,330 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:40,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:40,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:40,330 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:40,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:40,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:40,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1bea096782fcdd0d3636fed7e5e84b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:40,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:40,331 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111967 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:40,331 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/A is initiating minor compaction (all files) 2024-11-28T02:55:40,331 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/A in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,331 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7b531599a48248bb9b40430eb7e65137, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/071fb77facc348f69406630c5152e5e2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/edd251a0eb2a4023b38ebe7538343ffc] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=109.3 K 2024-11-28T02:55:40,331 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,331 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7b531599a48248bb9b40430eb7e65137, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/071fb77facc348f69406630c5152e5e2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/edd251a0eb2a4023b38ebe7538343ffc] 2024-11-28T02:55:40,331 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:40,331 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b531599a48248bb9b40430eb7e65137, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732762537667 2024-11-28T02:55:40,331 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/B is initiating minor compaction (all files) 2024-11-28T02:55:40,331 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/B in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,332 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/5c7278721b654b6aa0ddc27c2d9dc4fc, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/f864282431b34629b3babb966da3faa8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/eeab6c272d41483492f99d8573b22951] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=36.8 K 2024-11-28T02:55:40,332 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 071fb77facc348f69406630c5152e5e2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732762537750 2024-11-28T02:55:40,332 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting edd251a0eb2a4023b38ebe7538343ffc, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732762539956 2024-11-28T02:55:40,332 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c7278721b654b6aa0ddc27c2d9dc4fc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732762537667 2024-11-28T02:55:40,332 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting f864282431b34629b3babb966da3faa8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732762537750 2024-11-28T02:55:40,333 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting eeab6c272d41483492f99d8573b22951, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732762540035 2024-11-28T02:55:40,356 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:40,359 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#B#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:40,359 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/8ee1b7b5d74e4ea6a1e7892c299cb50b is 50, key is test_row_0/B:col10/1732762540035/Put/seqid=0 2024-11-28T02:55:40,368 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112832c825317c7c40e7b69207aecb038b7f_d1bea096782fcdd0d3636fed7e5e84b2 store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:40,370 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112832c825317c7c40e7b69207aecb038b7f_d1bea096782fcdd0d3636fed7e5e84b2, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:40,371 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112832c825317c7c40e7b69207aecb038b7f_d1bea096782fcdd0d3636fed7e5e84b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:40,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742252_1428 (size=4469) 2024-11-28T02:55:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:40,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-28T02:55:40,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:40,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:40,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:40,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:40,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:40,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:40,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742253_1429 (size=13221) 2024-11-28T02:55:40,422 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/8ee1b7b5d74e4ea6a1e7892c299cb50b as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/8ee1b7b5d74e4ea6a1e7892c299cb50b 2024-11-28T02:55:40,429 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/B of d1bea096782fcdd0d3636fed7e5e84b2 into 8ee1b7b5d74e4ea6a1e7892c299cb50b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:40,429 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:40,429 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/B, priority=13, startTime=1732762540330; duration=0sec 2024-11-28T02:55:40,429 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:40,429 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:B 2024-11-28T02:55:40,429 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:40,430 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:40,430 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): d1bea096782fcdd0d3636fed7e5e84b2/C is initiating minor compaction (all files) 2024-11-28T02:55:40,430 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1bea096782fcdd0d3636fed7e5e84b2/C in TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,431 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/a485bfae247c43059207ef72e66de090, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/f4f2ccdd12b54464870b2873379ce87e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/aa56e23c3a61409e99938dff4b05c102] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp, totalSize=36.8 K 2024-11-28T02:55:40,431 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting a485bfae247c43059207ef72e66de090, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732762537667 2024-11-28T02:55:40,431 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting f4f2ccdd12b54464870b2873379ce87e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732762537750 2024-11-28T02:55:40,432 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting aa56e23c3a61409e99938dff4b05c102, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732762540035 2024-11-28T02:55:40,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ddb9295579904d8f81721ed62c03d59c_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762540272/Put/seqid=0 2024-11-28T02:55:40,441 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#C#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:40,441 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/7ddf4f43b38b44d4bdd304c15ecade90 is 50, key is test_row_0/C:col10/1732762540035/Put/seqid=0 2024-11-28T02:55:40,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762600436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:40,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:40,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762600440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:40,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762600442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:40,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:40,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742254_1430 (size=14994) 2024-11-28T02:55:40,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742255_1431 (size=13221) 2024-11-28T02:55:40,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T02:55:40,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:40,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T02:55:40,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762600556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762600556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:40,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762600566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:40,619 INFO [Thread-1489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1489. 2024-11-28T02:55:40,619 DEBUG [Thread-1489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:40,619 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:40,619 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:40,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:40,624 INFO [Thread-1483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1483. 2024-11-28T02:55:40,624 DEBUG [Thread-1483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:40,624 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:40,624 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:40,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:40,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T02:55:40,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:40,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,632 INFO [Thread-1485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1485. 2024-11-28T02:55:40,632 DEBUG [Thread-1485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:40,632 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:40,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:40,632 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:40,633 INFO [Thread-1487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1487. 2024-11-28T02:55:40,633 DEBUG [Thread-1487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:40,633 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:40,633 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:40,633 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:40,640 INFO [Thread-1491 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1491. 2024-11-28T02:55:40,640 DEBUG [Thread-1491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:40,640 DEBUG [Thread-1491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:40,640 DEBUG [Thread-1491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:40,640 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:40,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:40,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:40,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762600776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762600776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762600776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:40,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:40,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T02:55:40,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:40,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T02:55:40,798 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1bea096782fcdd0d3636fed7e5e84b2#A#compaction#354 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:40,798 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/07eb14533c574dea96df751845d86ed0 is 175, key is test_row_0/A:col10/1732762540035/Put/seqid=0 2024-11-28T02:55:40,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742256_1432 (size=32175) 2024-11-28T02:55:40,861 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:40,864 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ddb9295579904d8f81721ed62c03d59c_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ddb9295579904d8f81721ed62c03d59c_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:40,865 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/f65b0f6d990c45518b6330ba0383f50b, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:40,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/f65b0f6d990c45518b6330ba0383f50b is 175, key is test_row_0/A:col10/1732762540272/Put/seqid=0 2024-11-28T02:55:40,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742257_1433 (size=39949) 2024-11-28T02:55:40,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=406, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/f65b0f6d990c45518b6330ba0383f50b 2024-11-28T02:55:40,875 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/7ddf4f43b38b44d4bdd304c15ecade90 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/7ddf4f43b38b44d4bdd304c15ecade90 2024-11-28T02:55:40,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/88ad2421a34f4b958c6f7210baf89635 is 50, key is test_row_0/B:col10/1732762540272/Put/seqid=0 2024-11-28T02:55:40,881 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/C of d1bea096782fcdd0d3636fed7e5e84b2 into 7ddf4f43b38b44d4bdd304c15ecade90(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:40,882 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:40,882 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/C, priority=13, startTime=1732762540330; duration=0sec 2024-11-28T02:55:40,882 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:40,882 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:C 2024-11-28T02:55:40,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742258_1434 (size=12301) 2024-11-28T02:55:40,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T02:55:40,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:40,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:40,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T02:55:41,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:41,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:41,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762601086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:41,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:41,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:41,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762601086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:41,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762601086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:41,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,208 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/07eb14533c574dea96df751845d86ed0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/07eb14533c574dea96df751845d86ed0 2024-11-28T02:55:41,213 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1bea096782fcdd0d3636fed7e5e84b2/A of d1bea096782fcdd0d3636fed7e5e84b2 into 07eb14533c574dea96df751845d86ed0(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:41,213 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:41,213 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., storeName=d1bea096782fcdd0d3636fed7e5e84b2/A, priority=13, startTime=1732762540330; duration=0sec 2024-11-28T02:55:41,213 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:41,213 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1bea096782fcdd0d3636fed7e5e84b2:A 2024-11-28T02:55:41,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T02:55:41,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:41,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:41,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:41,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/88ad2421a34f4b958c6f7210baf89635 2024-11-28T02:55:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T02:55:41,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/ff3f98f79ed242d4bf38d54a681ca4c5 is 50, key is test_row_0/C:col10/1732762540272/Put/seqid=0 2024-11-28T02:55:41,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742259_1435 (size=12301) 2024-11-28T02:55:41,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T02:55:41,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:41,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:41,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:41,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T02:55:41,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:41,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:41,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:41,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:41,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46002 deadline: 1732762601596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46010 deadline: 1732762601596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:41,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45982 deadline: 1732762601596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:55:41,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T02:55:41,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:41,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:41,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:41,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:41,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/ff3f98f79ed242d4bf38d54a681ca4c5 2024-11-28T02:55:41,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/f65b0f6d990c45518b6330ba0383f50b as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/f65b0f6d990c45518b6330ba0383f50b 2024-11-28T02:55:41,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/f65b0f6d990c45518b6330ba0383f50b, entries=200, sequenceid=406, filesize=39.0 K 2024-11-28T02:55:41,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/88ad2421a34f4b958c6f7210baf89635 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/88ad2421a34f4b958c6f7210baf89635 2024-11-28T02:55:41,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/88ad2421a34f4b958c6f7210baf89635, entries=150, sequenceid=406, filesize=12.0 K 2024-11-28T02:55:41,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/ff3f98f79ed242d4bf38d54a681ca4c5 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/ff3f98f79ed242d4bf38d54a681ca4c5 2024-11-28T02:55:41,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/ff3f98f79ed242d4bf38d54a681ca4c5, entries=150, sequenceid=406, filesize=12.0 K 2024-11-28T02:55:41,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for d1bea096782fcdd0d3636fed7e5e84b2 in 1323ms, sequenceid=406, compaction requested=false 2024-11-28T02:55:41,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:41,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-28T02:55:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:41,855 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-28T02:55:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:41,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:41,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ccc06f57e3f54316949b175466db6062_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762540424/Put/seqid=0 2024-11-28T02:55:41,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742260_1436 (size=12454) 2024-11-28T02:55:42,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:42,271 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ccc06f57e3f54316949b175466db6062_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ccc06f57e3f54316949b175466db6062_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:42,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/de4e87dd51984881ad2a46176ccf9602, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:42,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/de4e87dd51984881ad2a46176ccf9602 is 175, key is test_row_0/A:col10/1732762540424/Put/seqid=0 2024-11-28T02:55:42,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742261_1437 (size=31255) 2024-11-28T02:55:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T02:55:42,409 WARN [regionserver/978bd30859c3:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 5, running: 1 2024-11-28T02:55:42,415 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbba94a6f19e45ceab892a3ece3ba722, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7ae15e26c032457a83778cfbf48ec2aa, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/cdc5e47bab9b4c34b80128d3116cbf38, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/1241afa619654ae897c0bdbec8cdfa10, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/eab5cdcef3684316ae7aaf0ec92d91c7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/a19143f005684f0a829bd783c0ade653, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/3ae9d17168134cceaafd2069a1484c9a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7dcaff905d70469a8f0af29bb79a7d63, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ccdd1ab569a942098be82e4b85ee2880, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/a9e7b94f1b7c4327a30be45394358fa9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/b6968d12c2a04fc29a464ed4d494cd49, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e28794dda5e84150a849fbf616e50ef0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/252c6c6ab3ec4d2f9b92e7a100f6edaf, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/07e1d728dbc7440dabe63b98c292ff61, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ef1d037ee62c440f81f3a2138db46b13, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/c3f3a83c28f74fa98beabc1147ad6a70, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/f0baf3bd022e4743b8964bf73a6584c6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e574ab12cf194a5eba5d0cac2310a8bd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/40bd0f07ded5449291d438aea3e0658c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/9750ca32664d4c989a216a1b7b86f0a3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/4c3c56557c654cabac130384e0f23917, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/df9f496bc58140649191f7ff56ae0e8a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/02ca6971618d497480e3f41f332e7a21, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbaf969f2e3044ea976e6bfde3565cc8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7b531599a48248bb9b40430eb7e65137, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/071fb77facc348f69406630c5152e5e2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/edd251a0eb2a4023b38ebe7538343ffc] to archive 2024-11-28T02:55:42,416 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:55:42,420 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbba94a6f19e45ceab892a3ece3ba722 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbba94a6f19e45ceab892a3ece3ba722 2024-11-28T02:55:42,422 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7ae15e26c032457a83778cfbf48ec2aa to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7ae15e26c032457a83778cfbf48ec2aa 2024-11-28T02:55:42,423 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/cdc5e47bab9b4c34b80128d3116cbf38 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/cdc5e47bab9b4c34b80128d3116cbf38 2024-11-28T02:55:42,424 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/1241afa619654ae897c0bdbec8cdfa10 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/1241afa619654ae897c0bdbec8cdfa10 2024-11-28T02:55:42,425 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/eab5cdcef3684316ae7aaf0ec92d91c7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/eab5cdcef3684316ae7aaf0ec92d91c7 2024-11-28T02:55:42,426 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/a19143f005684f0a829bd783c0ade653 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/a19143f005684f0a829bd783c0ade653 2024-11-28T02:55:42,427 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/3ae9d17168134cceaafd2069a1484c9a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/3ae9d17168134cceaafd2069a1484c9a 2024-11-28T02:55:42,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7dcaff905d70469a8f0af29bb79a7d63 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7dcaff905d70469a8f0af29bb79a7d63 2024-11-28T02:55:42,430 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ccdd1ab569a942098be82e4b85ee2880 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ccdd1ab569a942098be82e4b85ee2880 2024-11-28T02:55:42,430 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/a9e7b94f1b7c4327a30be45394358fa9 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/a9e7b94f1b7c4327a30be45394358fa9 2024-11-28T02:55:42,431 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/b6968d12c2a04fc29a464ed4d494cd49 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/b6968d12c2a04fc29a464ed4d494cd49 2024-11-28T02:55:42,432 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e28794dda5e84150a849fbf616e50ef0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e28794dda5e84150a849fbf616e50ef0 2024-11-28T02:55:42,433 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/252c6c6ab3ec4d2f9b92e7a100f6edaf to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/252c6c6ab3ec4d2f9b92e7a100f6edaf 2024-11-28T02:55:42,434 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/07e1d728dbc7440dabe63b98c292ff61 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/07e1d728dbc7440dabe63b98c292ff61 2024-11-28T02:55:42,435 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ef1d037ee62c440f81f3a2138db46b13 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/ef1d037ee62c440f81f3a2138db46b13 2024-11-28T02:55:42,437 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/c3f3a83c28f74fa98beabc1147ad6a70 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/c3f3a83c28f74fa98beabc1147ad6a70 2024-11-28T02:55:42,438 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/f0baf3bd022e4743b8964bf73a6584c6 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/f0baf3bd022e4743b8964bf73a6584c6 2024-11-28T02:55:42,439 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e574ab12cf194a5eba5d0cac2310a8bd to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/e574ab12cf194a5eba5d0cac2310a8bd 2024-11-28T02:55:42,440 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/40bd0f07ded5449291d438aea3e0658c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/40bd0f07ded5449291d438aea3e0658c 2024-11-28T02:55:42,441 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/9750ca32664d4c989a216a1b7b86f0a3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/9750ca32664d4c989a216a1b7b86f0a3 2024-11-28T02:55:42,442 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/4c3c56557c654cabac130384e0f23917 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/4c3c56557c654cabac130384e0f23917 2024-11-28T02:55:42,443 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/df9f496bc58140649191f7ff56ae0e8a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/df9f496bc58140649191f7ff56ae0e8a 2024-11-28T02:55:42,444 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/02ca6971618d497480e3f41f332e7a21 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/02ca6971618d497480e3f41f332e7a21 2024-11-28T02:55:42,446 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbaf969f2e3044ea976e6bfde3565cc8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/dbaf969f2e3044ea976e6bfde3565cc8 2024-11-28T02:55:42,447 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7b531599a48248bb9b40430eb7e65137 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/7b531599a48248bb9b40430eb7e65137 2024-11-28T02:55:42,448 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/071fb77facc348f69406630c5152e5e2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/071fb77facc348f69406630c5152e5e2 2024-11-28T02:55:42,449 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/edd251a0eb2a4023b38ebe7538343ffc to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/edd251a0eb2a4023b38ebe7538343ffc 2024-11-28T02:55:42,455 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/c7e6b79deeae4cedaa2911f369b20ebe, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/2197ea7d18c04c69b64b8dd76686eb81, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/aebc85e7631c4e6f955fbfdca538cdf3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/5199f7a374a24911b87600e7bc7cc3c6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/07c687a2adf74f889491796c75084d38, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/149d3925077f4a3dbb7095c6173f2f26, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/36de86748cd24c359bdf2670b0c6bb67, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/540e6caa07844367a7b9091359baad31, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/684b5904fef94a309bf6edcb1e43503d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/3382e01b716c4530b7dac5da662154af, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/d114cce9550b4efca98e2545fe66750d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/bbd3287f1db9495c80fbfebc81b3f89c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/9ea64dc891dc4d41b17044320f3340ec, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/ee58ef245489405f9ed9abf1c691215d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/0d2c22ec24a348d79e2064db3e845ffb, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/aa9fdf72bc4e48e6b743fbeeaa18a5a3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/6dff697cf185444cb8526c2c742252fa, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/b63234a1f0ea470697e77bd24024c501, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/8faeec23e63e4405b51a27112b155953, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/9fbafd4fade145c881e3b2d3aa8457bd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/896c541fa1bd41f5b355453f89ac096c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/75910f943f724725a4da957f8d82caf7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/41b149fde1ac4621a8dbcb53458559c8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/5c7278721b654b6aa0ddc27c2d9dc4fc, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/4623f529326747e69a9df56873e5617e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/f864282431b34629b3babb966da3faa8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/eeab6c272d41483492f99d8573b22951] to archive 2024-11-28T02:55:42,456 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:55:42,457 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/c7e6b79deeae4cedaa2911f369b20ebe to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/c7e6b79deeae4cedaa2911f369b20ebe 2024-11-28T02:55:42,458 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/2197ea7d18c04c69b64b8dd76686eb81 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/2197ea7d18c04c69b64b8dd76686eb81 2024-11-28T02:55:42,459 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/aebc85e7631c4e6f955fbfdca538cdf3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/aebc85e7631c4e6f955fbfdca538cdf3 2024-11-28T02:55:42,460 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/5199f7a374a24911b87600e7bc7cc3c6 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/5199f7a374a24911b87600e7bc7cc3c6 2024-11-28T02:55:42,461 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/07c687a2adf74f889491796c75084d38 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/07c687a2adf74f889491796c75084d38 2024-11-28T02:55:42,462 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/149d3925077f4a3dbb7095c6173f2f26 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/149d3925077f4a3dbb7095c6173f2f26 2024-11-28T02:55:42,463 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/36de86748cd24c359bdf2670b0c6bb67 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/36de86748cd24c359bdf2670b0c6bb67 2024-11-28T02:55:42,464 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/540e6caa07844367a7b9091359baad31 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/540e6caa07844367a7b9091359baad31 2024-11-28T02:55:42,465 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/684b5904fef94a309bf6edcb1e43503d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/684b5904fef94a309bf6edcb1e43503d 2024-11-28T02:55:42,465 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/3382e01b716c4530b7dac5da662154af to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/3382e01b716c4530b7dac5da662154af 2024-11-28T02:55:42,467 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/d114cce9550b4efca98e2545fe66750d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/d114cce9550b4efca98e2545fe66750d 2024-11-28T02:55:42,468 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/bbd3287f1db9495c80fbfebc81b3f89c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/bbd3287f1db9495c80fbfebc81b3f89c 2024-11-28T02:55:42,469 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/9ea64dc891dc4d41b17044320f3340ec to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/9ea64dc891dc4d41b17044320f3340ec 2024-11-28T02:55:42,470 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/ee58ef245489405f9ed9abf1c691215d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/ee58ef245489405f9ed9abf1c691215d 2024-11-28T02:55:42,471 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/0d2c22ec24a348d79e2064db3e845ffb to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/0d2c22ec24a348d79e2064db3e845ffb 2024-11-28T02:55:42,472 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/aa9fdf72bc4e48e6b743fbeeaa18a5a3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/aa9fdf72bc4e48e6b743fbeeaa18a5a3 2024-11-28T02:55:42,473 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/6dff697cf185444cb8526c2c742252fa to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/6dff697cf185444cb8526c2c742252fa 2024-11-28T02:55:42,474 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/b63234a1f0ea470697e77bd24024c501 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/b63234a1f0ea470697e77bd24024c501 2024-11-28T02:55:42,475 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/8faeec23e63e4405b51a27112b155953 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/8faeec23e63e4405b51a27112b155953 2024-11-28T02:55:42,476 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/9fbafd4fade145c881e3b2d3aa8457bd to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/9fbafd4fade145c881e3b2d3aa8457bd 2024-11-28T02:55:42,477 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/896c541fa1bd41f5b355453f89ac096c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/896c541fa1bd41f5b355453f89ac096c 2024-11-28T02:55:42,478 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/75910f943f724725a4da957f8d82caf7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/75910f943f724725a4da957f8d82caf7 2024-11-28T02:55:42,479 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/41b149fde1ac4621a8dbcb53458559c8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/41b149fde1ac4621a8dbcb53458559c8 2024-11-28T02:55:42,480 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/5c7278721b654b6aa0ddc27c2d9dc4fc to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/5c7278721b654b6aa0ddc27c2d9dc4fc 2024-11-28T02:55:42,481 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/4623f529326747e69a9df56873e5617e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/4623f529326747e69a9df56873e5617e 2024-11-28T02:55:42,482 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/f864282431b34629b3babb966da3faa8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/f864282431b34629b3babb966da3faa8 2024-11-28T02:55:42,483 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/eeab6c272d41483492f99d8573b22951 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/eeab6c272d41483492f99d8573b22951 2024-11-28T02:55:42,487 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/b3839927eb2f4f4ab2205ea2de67508f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/11d014bb28f0455eac62fdeef87a9ea4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/29c9d8d58baf4b1b8b6fa450c2e3ab20, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/d855613a7f9d4752b49e497155a3fd56, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8da9a4870767496484fa6b84844a29f0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/a83fab5c1aa24cfda1179ce3ccd0f20b, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/42d1a4aafb524788874d6ee92b2c043c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8217ac0310e04dd3ab293ee6434f8205, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/6ec7839e5858448fb5f5885a6d91a9d8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/9dd5538f88e74a07ba35de529610d1ca, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/fe2de5a7c87b4c09b6cb469e6b07583a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8d7893afd15547b28aae12aa143dff29, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/aa97d8ff86eb4234a691c7e07347de86, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/df1f191091c54144bcba3e52477436a3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/cb194a805ed5436f95a23305f9707d85, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/e2bb255128834e55b1305628dd70fd36, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/81e1709adfc340cea5c76ac86655241f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/2f889ec855b24dcea9206403e4219289, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/2dbd77cc46704a65bc94e912740156f9, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/0116af6c17904137b23175a696132c8d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/21d654593fb244dbb9fd6cd304e6c002, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/1c3abbf192d14bf09f19bd24d064909e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/f0058726fa384c568ddb8ccb0303a105, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/a485bfae247c43059207ef72e66de090, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/b2f164adbf3c4a0caf46daafdff1988e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/f4f2ccdd12b54464870b2873379ce87e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/aa56e23c3a61409e99938dff4b05c102] to archive 2024-11-28T02:55:42,488 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:55:42,489 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/b3839927eb2f4f4ab2205ea2de67508f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/b3839927eb2f4f4ab2205ea2de67508f 2024-11-28T02:55:42,490 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/11d014bb28f0455eac62fdeef87a9ea4 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/11d014bb28f0455eac62fdeef87a9ea4 2024-11-28T02:55:42,492 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/29c9d8d58baf4b1b8b6fa450c2e3ab20 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/29c9d8d58baf4b1b8b6fa450c2e3ab20 2024-11-28T02:55:42,493 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/d855613a7f9d4752b49e497155a3fd56 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/d855613a7f9d4752b49e497155a3fd56 2024-11-28T02:55:42,494 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8da9a4870767496484fa6b84844a29f0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8da9a4870767496484fa6b84844a29f0 2024-11-28T02:55:42,495 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/a83fab5c1aa24cfda1179ce3ccd0f20b to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/a83fab5c1aa24cfda1179ce3ccd0f20b 2024-11-28T02:55:42,496 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/42d1a4aafb524788874d6ee92b2c043c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/42d1a4aafb524788874d6ee92b2c043c 2024-11-28T02:55:42,497 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8217ac0310e04dd3ab293ee6434f8205 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8217ac0310e04dd3ab293ee6434f8205 2024-11-28T02:55:42,498 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/6ec7839e5858448fb5f5885a6d91a9d8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/6ec7839e5858448fb5f5885a6d91a9d8 2024-11-28T02:55:42,499 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/9dd5538f88e74a07ba35de529610d1ca to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/9dd5538f88e74a07ba35de529610d1ca 2024-11-28T02:55:42,500 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/fe2de5a7c87b4c09b6cb469e6b07583a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/fe2de5a7c87b4c09b6cb469e6b07583a 2024-11-28T02:55:42,501 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8d7893afd15547b28aae12aa143dff29 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/8d7893afd15547b28aae12aa143dff29 2024-11-28T02:55:42,503 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/aa97d8ff86eb4234a691c7e07347de86 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/aa97d8ff86eb4234a691c7e07347de86 2024-11-28T02:55:42,504 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/df1f191091c54144bcba3e52477436a3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/df1f191091c54144bcba3e52477436a3 2024-11-28T02:55:42,505 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/cb194a805ed5436f95a23305f9707d85 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/cb194a805ed5436f95a23305f9707d85 2024-11-28T02:55:42,506 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/e2bb255128834e55b1305628dd70fd36 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/e2bb255128834e55b1305628dd70fd36 2024-11-28T02:55:42,507 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/81e1709adfc340cea5c76ac86655241f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/81e1709adfc340cea5c76ac86655241f 2024-11-28T02:55:42,508 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/2f889ec855b24dcea9206403e4219289 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/2f889ec855b24dcea9206403e4219289 2024-11-28T02:55:42,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/2dbd77cc46704a65bc94e912740156f9 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/2dbd77cc46704a65bc94e912740156f9 2024-11-28T02:55:42,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/0116af6c17904137b23175a696132c8d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/0116af6c17904137b23175a696132c8d 2024-11-28T02:55:42,510 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/21d654593fb244dbb9fd6cd304e6c002 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/21d654593fb244dbb9fd6cd304e6c002 2024-11-28T02:55:42,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/1c3abbf192d14bf09f19bd24d064909e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/1c3abbf192d14bf09f19bd24d064909e 2024-11-28T02:55:42,512 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/f0058726fa384c568ddb8ccb0303a105 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/f0058726fa384c568ddb8ccb0303a105 2024-11-28T02:55:42,513 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/a485bfae247c43059207ef72e66de090 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/a485bfae247c43059207ef72e66de090 2024-11-28T02:55:42,513 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/b2f164adbf3c4a0caf46daafdff1988e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/b2f164adbf3c4a0caf46daafdff1988e 2024-11-28T02:55:42,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/f4f2ccdd12b54464870b2873379ce87e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/f4f2ccdd12b54464870b2873379ce87e 2024-11-28T02:55:42,515 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/aa56e23c3a61409e99938dff4b05c102 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/aa56e23c3a61409e99938dff4b05c102 2024-11-28T02:55:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:42,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. as already flushing 2024-11-28T02:55:42,608 INFO [Thread-1472 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1472. 2024-11-28T02:55:42,608 INFO [Thread-1480 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1480. 2024-11-28T02:55:42,608 DEBUG [Thread-1472 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:42,608 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:42,608 DEBUG [Thread-1480 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:42,608 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:42,608 INFO [Thread-1478 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1478. 2024-11-28T02:55:42,608 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:42,608 DEBUG [Thread-1478 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:42,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:42,609 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:42,609 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:42,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:42,609 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:42,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:42,680 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=418, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/de4e87dd51984881ad2a46176ccf9602 2024-11-28T02:55:42,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/cf8a012971964f15a24202df06e25449 is 50, key is test_row_0/B:col10/1732762540424/Put/seqid=0 2024-11-28T02:55:42,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742262_1438 (size=12301) 2024-11-28T02:55:43,098 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/cf8a012971964f15a24202df06e25449 2024-11-28T02:55:43,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/357fb25b55dd4212b6f5c2bedb29637e is 50, key is test_row_0/C:col10/1732762540424/Put/seqid=0 2024-11-28T02:55:43,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742263_1439 (size=12301) 2024-11-28T02:55:43,512 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/357fb25b55dd4212b6f5c2bedb29637e 2024-11-28T02:55:43,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/de4e87dd51984881ad2a46176ccf9602 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/de4e87dd51984881ad2a46176ccf9602 2024-11-28T02:55:43,528 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/de4e87dd51984881ad2a46176ccf9602, entries=150, sequenceid=418, filesize=30.5 K 2024-11-28T02:55:43,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/cf8a012971964f15a24202df06e25449 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/cf8a012971964f15a24202df06e25449 2024-11-28T02:55:43,532 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/cf8a012971964f15a24202df06e25449, entries=150, sequenceid=418, filesize=12.0 K 2024-11-28T02:55:43,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/357fb25b55dd4212b6f5c2bedb29637e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/357fb25b55dd4212b6f5c2bedb29637e 2024-11-28T02:55:43,535 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/357fb25b55dd4212b6f5c2bedb29637e, entries=150, sequenceid=418, filesize=12.0 K 2024-11-28T02:55:43,536 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=20.13 KB/20610 for d1bea096782fcdd0d3636fed7e5e84b2 in 1681ms, sequenceid=418, compaction requested=true 2024-11-28T02:55:43,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for d1bea096782fcdd0d3636fed7e5e84b2: 2024-11-28T02:55:43,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:43,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-28T02:55:43,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-28T02:55:43,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-28T02:55:43,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3780 sec 2024-11-28T02:55:43,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 3.3840 sec 2024-11-28T02:55:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-28T02:55:44,326 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:44,519 INFO [Thread-1476 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1476. 2024-11-28T02:55:44,519 DEBUG [Thread-1476 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:44,519 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:44,519 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:44,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:44,528 INFO [Thread-1474 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1474. 2024-11-28T02:55:44,528 DEBUG [Thread-1474 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:55:44,528 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:44,528 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:44,528 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 896 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2687 rows 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 903 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2704 rows 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 906 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2716 rows 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 905 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2714 rows 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 902 2024-11-28T02:55:44,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2704 rows 2024-11-28T02:55:44,529 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:55:44,529 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:55:44,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:44,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:44,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:55:44,530 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T02:55:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T02:55:44,533 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762544533"}]},"ts":"1732762544533"} 2024-11-28T02:55:44,534 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T02:55:44,534 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T02:55:44,534 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T02:55:44,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, UNASSIGN}] 2024-11-28T02:55:44,536 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, UNASSIGN 2024-11-28T02:55:44,537 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=d1bea096782fcdd0d3636fed7e5e84b2, regionState=CLOSING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:55:44,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, UNASSIGN because future has completed 2024-11-28T02:55:44,538 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T02:55:44,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:55:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T02:55:44,691 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(122): Close d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:44,691 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T02:55:44,691 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1722): Closing d1bea096782fcdd0d3636fed7e5e84b2, disabling compactions & flushes 2024-11-28T02:55:44,691 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:44,691 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:44,691 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. after waiting 0 ms 2024-11-28T02:55:44,691 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:44,691 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(2902): Flushing d1bea096782fcdd0d3636fed7e5e84b2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T02:55:44,691 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=A 2024-11-28T02:55:44,691 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:44,691 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=B 2024-11-28T02:55:44,691 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:44,692 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1bea096782fcdd0d3636fed7e5e84b2, store=C 2024-11-28T02:55:44,692 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:44,697 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287adfb2369dfe4f93ae929bfb849bd89b_d1bea096782fcdd0d3636fed7e5e84b2 is 50, key is test_row_0/A:col10/1732762544516/Put/seqid=0 2024-11-28T02:55:44,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742264_1440 (size=12454) 2024-11-28T02:55:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T02:55:45,100 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:45,104 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287adfb2369dfe4f93ae929bfb849bd89b_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287adfb2369dfe4f93ae929bfb849bd89b_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:45,105 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/33e2f36273314659b6e5c6b840cda4eb, store: [table=TestAcidGuarantees family=A region=d1bea096782fcdd0d3636fed7e5e84b2] 2024-11-28T02:55:45,105 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/33e2f36273314659b6e5c6b840cda4eb is 175, key is test_row_0/A:col10/1732762544516/Put/seqid=0 2024-11-28T02:55:45,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742265_1441 (size=31255) 2024-11-28T02:55:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T02:55:45,510 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=426, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/33e2f36273314659b6e5c6b840cda4eb 2024-11-28T02:55:45,520 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/d60c445048494c50aa3efb243e61e8f8 is 50, key is test_row_0/B:col10/1732762544516/Put/seqid=0 2024-11-28T02:55:45,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742266_1442 (size=12301) 2024-11-28T02:55:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T02:55:45,924 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/d60c445048494c50aa3efb243e61e8f8 2024-11-28T02:55:45,932 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/0ffdfa7a9bf24ec083a7a501e0dcdb0b is 50, key is test_row_0/C:col10/1732762544516/Put/seqid=0 2024-11-28T02:55:45,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742267_1443 (size=12301) 2024-11-28T02:55:46,336 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/0ffdfa7a9bf24ec083a7a501e0dcdb0b 2024-11-28T02:55:46,342 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/A/33e2f36273314659b6e5c6b840cda4eb as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/33e2f36273314659b6e5c6b840cda4eb 2024-11-28T02:55:46,347 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/33e2f36273314659b6e5c6b840cda4eb, entries=150, sequenceid=426, filesize=30.5 K 2024-11-28T02:55:46,348 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/B/d60c445048494c50aa3efb243e61e8f8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/d60c445048494c50aa3efb243e61e8f8 2024-11-28T02:55:46,352 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/d60c445048494c50aa3efb243e61e8f8, entries=150, sequenceid=426, filesize=12.0 K 2024-11-28T02:55:46,353 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/.tmp/C/0ffdfa7a9bf24ec083a7a501e0dcdb0b as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/0ffdfa7a9bf24ec083a7a501e0dcdb0b 2024-11-28T02:55:46,357 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/0ffdfa7a9bf24ec083a7a501e0dcdb0b, entries=150, sequenceid=426, filesize=12.0 K 2024-11-28T02:55:46,358 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d1bea096782fcdd0d3636fed7e5e84b2 in 1667ms, sequenceid=426, compaction requested=true 2024-11-28T02:55:46,363 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/recovered.edits/429.seqid, newMaxSeqId=429, maxSeqId=4 2024-11-28T02:55:46,363 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. 2024-11-28T02:55:46,363 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1676): Region close journal for d1bea096782fcdd0d3636fed7e5e84b2: Waiting for close lock at 1732762544691Running coprocessor pre-close hooks at 1732762544691Disabling compacts and flushes for region at 1732762544691Disabling writes for close at 1732762544691Obtaining lock to block concurrent updates at 1732762544691Preparing flush snapshotting stores in d1bea096782fcdd0d3636fed7e5e84b2 at 1732762544691Finished memstore snapshotting TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732762544692 (+1 ms)Flushing stores of TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2. at 1732762544692Flushing d1bea096782fcdd0d3636fed7e5e84b2/A: creating writer at 1732762544692Flushing mob file d1bea096782fcdd0d3636fed7e5e84b2/A: appending metadata at 1732762544696 (+4 ms)Flushing mob file d1bea096782fcdd0d3636fed7e5e84b2/A: closing flushed file at 1732762544697 (+1 ms)Flushing d1bea096782fcdd0d3636fed7e5e84b2/A: appending metadata at 1732762545105 (+408 ms)Flushing d1bea096782fcdd0d3636fed7e5e84b2/A: closing flushed file at 1732762545105Flushing d1bea096782fcdd0d3636fed7e5e84b2/B: creating writer at 1732762545516 (+411 ms)Flushing d1bea096782fcdd0d3636fed7e5e84b2/B: appending metadata at 1732762545519 (+3 ms)Flushing d1bea096782fcdd0d3636fed7e5e84b2/B: closing flushed file at 1732762545519Flushing d1bea096782fcdd0d3636fed7e5e84b2/C: creating writer at 1732762545929 (+410 ms)Flushing d1bea096782fcdd0d3636fed7e5e84b2/C: appending metadata at 1732762545932 (+3 ms)Flushing d1bea096782fcdd0d3636fed7e5e84b2/C: closing flushed file at 1732762545932Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@474c91d1: reopening flushed file at 1732762546341 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68950d3c: reopening flushed file at 1732762546347 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c6416c1: reopening flushed file at 1732762546352 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d1bea096782fcdd0d3636fed7e5e84b2 in 1667ms, sequenceid=426, compaction requested=true at 1732762546358 (+6 ms)Writing region close event to WAL at 1732762546360 (+2 ms)Running coprocessor post-close hooks at 1732762546363 (+3 ms)Closed at 1732762546363 2024-11-28T02:55:46,365 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(157): Closed d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,366 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=d1bea096782fcdd0d3636fed7e5e84b2, regionState=CLOSED 2024-11-28T02:55:46,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:55:46,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-28T02:55:46,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseRegionProcedure d1bea096782fcdd0d3636fed7e5e84b2, server=978bd30859c3,41631,1732762419946 in 1.8290 sec 2024-11-28T02:55:46,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-28T02:55:46,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1bea096782fcdd0d3636fed7e5e84b2, UNASSIGN in 1.8330 sec 2024-11-28T02:55:46,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-28T02:55:46,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8370 sec 2024-11-28T02:55:46,373 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762546373"}]},"ts":"1732762546373"} 2024-11-28T02:55:46,374 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T02:55:46,374 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T02:55:46,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8450 sec 2024-11-28T02:55:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T02:55:46,677 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:46,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T02:55:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:46,682 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T02:55:46,684 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=121, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:46,688 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,693 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/recovered.edits] 2024-11-28T02:55:46,698 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/07eb14533c574dea96df751845d86ed0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/07eb14533c574dea96df751845d86ed0 2024-11-28T02:55:46,700 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/33e2f36273314659b6e5c6b840cda4eb to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/33e2f36273314659b6e5c6b840cda4eb 2024-11-28T02:55:46,702 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/de4e87dd51984881ad2a46176ccf9602 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/de4e87dd51984881ad2a46176ccf9602 2024-11-28T02:55:46,704 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/f65b0f6d990c45518b6330ba0383f50b to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/A/f65b0f6d990c45518b6330ba0383f50b 2024-11-28T02:55:46,706 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/88ad2421a34f4b958c6f7210baf89635 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/88ad2421a34f4b958c6f7210baf89635 2024-11-28T02:55:46,708 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/8ee1b7b5d74e4ea6a1e7892c299cb50b to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/8ee1b7b5d74e4ea6a1e7892c299cb50b 2024-11-28T02:55:46,709 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/cf8a012971964f15a24202df06e25449 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/cf8a012971964f15a24202df06e25449 2024-11-28T02:55:46,710 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/d60c445048494c50aa3efb243e61e8f8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/B/d60c445048494c50aa3efb243e61e8f8 2024-11-28T02:55:46,713 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/0ffdfa7a9bf24ec083a7a501e0dcdb0b to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/0ffdfa7a9bf24ec083a7a501e0dcdb0b 2024-11-28T02:55:46,714 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/357fb25b55dd4212b6f5c2bedb29637e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/357fb25b55dd4212b6f5c2bedb29637e 2024-11-28T02:55:46,715 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/7ddf4f43b38b44d4bdd304c15ecade90 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/7ddf4f43b38b44d4bdd304c15ecade90 2024-11-28T02:55:46,716 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/ff3f98f79ed242d4bf38d54a681ca4c5 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/C/ff3f98f79ed242d4bf38d54a681ca4c5 2024-11-28T02:55:46,718 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/recovered.edits/429.seqid to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2/recovered.edits/429.seqid 2024-11-28T02:55:46,719 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,719 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T02:55:46,719 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T02:55:46,720 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T02:55:46,722 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112803227fafa1fe46ffaac4b1f62dc6ecc3_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112803227fafa1fe46ffaac4b1f62dc6ecc3_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,723 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280541406c7ae144a3a64397df1d48996d_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280541406c7ae144a3a64397df1d48996d_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,724 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280dc645ef66e84f159dff53f99ce90a8a_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280dc645ef66e84f159dff53f99ce90a8a_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,725 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282096c224cef84c92aeb1efacb6ffea40_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282096c224cef84c92aeb1efacb6ffea40_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,725 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283b15a869fb7d41f6b5887aba8cb67e3b_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283b15a869fb7d41f6b5887aba8cb67e3b_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,726 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285c4d54283c344ef5b05fc06f4eba129f_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285c4d54283c344ef5b05fc06f4eba129f_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,727 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285f7bedb99d5647129181c21d4f033512_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285f7bedb99d5647129181c21d4f033512_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,728 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112870da7db30fa743c09e85b054fb49c805_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112870da7db30fa743c09e85b054fb49c805_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,729 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112875ebba8d7b22472ca2f1cf65a8e38e2f_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112875ebba8d7b22472ca2f1cf65a8e38e2f_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,730 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287adfb2369dfe4f93ae929bfb849bd89b_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287adfb2369dfe4f93ae929bfb849bd89b_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,730 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112882ca9aa5993c4a27b4839400d310124f_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112882ca9aa5993c4a27b4839400d310124f_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,731 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288787e0422ebc4b89858ecdccdd278a4b_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288787e0422ebc4b89858ecdccdd278a4b_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,732 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112887f44bd032414762b44b186447992e85_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112887f44bd032414762b44b186447992e85_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,732 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288e96eac325b9470499684b8e252347a1_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288e96eac325b9470499684b8e252347a1_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,733 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a13c047ec42e4dbd99bf673a9181e9c3_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a13c047ec42e4dbd99bf673a9181e9c3_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,734 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a205b5494e354c8193a4fa4f66b4147e_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a205b5494e354c8193a4fa4f66b4147e_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,734 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c13bb4c7bebd4c73a87b05a64e0a5b57_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c13bb4c7bebd4c73a87b05a64e0a5b57_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,735 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c7c382180131466fa741aeb85baf38cc_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c7c382180131466fa741aeb85baf38cc_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,735 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ccc06f57e3f54316949b175466db6062_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ccc06f57e3f54316949b175466db6062_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,736 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d230a2748ebe402c94cee1c29abdb178_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d230a2748ebe402c94cee1c29abdb178_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,737 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ddb9295579904d8f81721ed62c03d59c_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ddb9295579904d8f81721ed62c03d59c_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,737 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fe07f1122d3f4b50bf1e5ea94627bcc5_d1bea096782fcdd0d3636fed7e5e84b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fe07f1122d3f4b50bf1e5ea94627bcc5_d1bea096782fcdd0d3636fed7e5e84b2 2024-11-28T02:55:46,738 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T02:55:46,739 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=121, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:46,741 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T02:55:46,743 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T02:55:46,744 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=121, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:46,744 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T02:55:46,744 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732762546744"}]},"ts":"9223372036854775807"} 2024-11-28T02:55:46,746 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T02:55:46,746 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d1bea096782fcdd0d3636fed7e5e84b2, NAME => 'TestAcidGuarantees,,1732762517444.d1bea096782fcdd0d3636fed7e5e84b2.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T02:55:46,746 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T02:55:46,746 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732762546746"}]},"ts":"9223372036854775807"} 2024-11-28T02:55:46,748 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T02:55:46,748 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=121, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:46,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-28T02:55:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T02:55:46,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T02:55:46,787 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:46,796 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 246) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1299054995_22 at /127.0.0.1:32986 [Waiting for operation #1577] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1042822658_22 at /127.0.0.1:34856 [Waiting for operation #1527] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/978bd30859c3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1299054995_22 at /127.0.0.1:34842 [Waiting for operation #1543] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1042822658_22 at /127.0.0.1:34868 [Waiting for operation #1547] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=540 (was 429) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7435 (was 7197) - AvailableMemoryMB LEAK? - 2024-11-28T02:55:46,804 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=540, ProcessCount=11, AvailableMemoryMB=7435 2024-11-28T02:55:46,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T02:55:46,806 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:55:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T02:55:46,807 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T02:55:46,807 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:46,807 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 122 2024-11-28T02:55:46,808 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T02:55:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-28T02:55:46,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742268_1444 (size=1061) 2024-11-28T02:55:46,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-28T02:55:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-28T02:55:47,221 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381 2024-11-28T02:55:47,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742269_1445 (size=53) 2024-11-28T02:55:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-28T02:55:47,631 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:55:47,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2a9027925ba3f01c5e0048c9e37ce432, disabling compactions & flushes 2024-11-28T02:55:47,632 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:47,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:47,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. after waiting 0 ms 2024-11-28T02:55:47,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:47,632 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:47,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2a9027925ba3f01c5e0048c9e37ce432: Waiting for close lock at 1732762547632Disabling compacts and flushes for region at 1732762547632Disabling writes for close at 1732762547632Writing region close event to WAL at 1732762547632Closed at 1732762547632 2024-11-28T02:55:47,634 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T02:55:47,634 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732762547634"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732762547634"}]},"ts":"1732762547634"} 2024-11-28T02:55:47,637 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T02:55:47,639 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T02:55:47,639 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762547639"}]},"ts":"1732762547639"} 2024-11-28T02:55:47,642 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T02:55:47,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a9027925ba3f01c5e0048c9e37ce432, ASSIGN}] 2024-11-28T02:55:47,644 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a9027925ba3f01c5e0048c9e37ce432, ASSIGN 2024-11-28T02:55:47,646 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a9027925ba3f01c5e0048c9e37ce432, ASSIGN; state=OFFLINE, location=978bd30859c3,41631,1732762419946; forceNewPlan=false, retain=false 2024-11-28T02:55:47,797 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=2a9027925ba3f01c5e0048c9e37ce432, regionState=OPENING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:55:47,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a9027925ba3f01c5e0048c9e37ce432, ASSIGN because future has completed 2024-11-28T02:55:47,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:55:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-28T02:55:47,959 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:47,960 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:55:47,961 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:47,961 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:55:47,961 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7794): checking encryption for 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:47,961 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7797): checking classloading for 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:47,965 INFO [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:47,967 INFO [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:55:47,967 INFO [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a9027925ba3f01c5e0048c9e37ce432 columnFamilyName A 2024-11-28T02:55:47,967 DEBUG [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:47,967 INFO [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] regionserver.HStore(327): Store=2a9027925ba3f01c5e0048c9e37ce432/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:55:47,968 INFO [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:47,968 INFO [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:55:47,969 INFO [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a9027925ba3f01c5e0048c9e37ce432 columnFamilyName B 2024-11-28T02:55:47,969 DEBUG [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:47,969 INFO [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] regionserver.HStore(327): Store=2a9027925ba3f01c5e0048c9e37ce432/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:55:47,969 INFO [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:47,970 INFO [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:55:47,970 INFO [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a9027925ba3f01c5e0048c9e37ce432 columnFamilyName C 2024-11-28T02:55:47,970 DEBUG [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:55:47,970 INFO [StoreOpener-2a9027925ba3f01c5e0048c9e37ce432-1 {}] regionserver.HStore(327): Store=2a9027925ba3f01c5e0048c9e37ce432/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:55:47,971 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:47,971 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1038): replaying wal for 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:47,971 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:47,971 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:47,972 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1048): stopping wal replay for 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:47,972 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1060): Cleaning up temporary data for 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:47,973 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T02:55:47,973 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1093): writing seq id for 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:47,975 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:55:47,975 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1114): Opened 2a9027925ba3f01c5e0048c9e37ce432; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66189079, jitterRate=-0.013705864548683167}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T02:55:47,975 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:47,976 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1006): Region open journal for 2a9027925ba3f01c5e0048c9e37ce432: Running coprocessor pre-open hook at 1732762547962Writing region info on filesystem at 1732762547962Initializing all the Stores at 1732762547964 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762547964Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762547964Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762547965 (+1 ms)Cleaning up temporary data from old regions at 1732762547972 (+7 ms)Running coprocessor post-open hooks at 1732762547975 (+3 ms)Region opened successfully at 1732762547976 (+1 ms) 2024-11-28T02:55:47,977 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., pid=124, masterSystemTime=1732762547954 2024-11-28T02:55:47,978 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:47,978 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:47,979 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=2a9027925ba3f01c5e0048c9e37ce432, regionState=OPEN, openSeqNum=2, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:55:47,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:55:47,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-28T02:55:47,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 in 181 msec 2024-11-28T02:55:47,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-28T02:55:47,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a9027925ba3f01c5e0048c9e37ce432, ASSIGN in 340 msec 2024-11-28T02:55:47,984 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T02:55:47,984 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762547984"}]},"ts":"1732762547984"} 2024-11-28T02:55:47,986 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T02:55:47,986 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T02:55:47,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-28T02:55:48,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-28T02:55:48,956 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:48,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ded0f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:48,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:48,959 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:48,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:48,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:48,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6459553a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:48,960 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ff8db8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,961 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:48,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:48,964 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T02:55:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 978bd30859c3,32823,1732762419008 2024-11-28T02:55:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@52fb2022 2024-11-28T02:55:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T02:55:48,965 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T02:55:48,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:48,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:48,967 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33e2f95c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:48,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:48,968 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@588d5197, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:48,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:48,970 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:48,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:48,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:48,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@570e1904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:48,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:48,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:48,971 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:48,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@203246c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:48,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:48,973 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30595f81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:48,974 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:48,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17381a35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,976 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:48,976 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:48,977 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48f96ffa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:48,978 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:48,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9bb5090, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:48,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:48,983 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59a55fa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:48,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:48,984 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:48,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c858a10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:48,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:48,986 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@131674aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:48,987 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:48,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@498165a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:48,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:48,989 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:48,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:48,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11764a2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:48,990 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:48,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@644ecbd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:48,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:48,992 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:48,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:48,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:48,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3728be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:48,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:48,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:48,993 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef57d59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:48,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:48,995 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@482df146, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:48,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:48,997 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@403cd2bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:49,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:49,000 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:55:49,000 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:55:49,000 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:55:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:55:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:55:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a85276f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:55:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:55:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:55:49,002 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:55:49,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34cc8f68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:55:49,003 DEBUG [Thread-2007 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:49,003 DEBUG [Thread-2022 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:49,003 DEBUG [Thread-2018 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:49,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:49,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:49,004 DEBUG [Thread-2011 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:49,004 DEBUG [Thread-2009 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:49,004 DEBUG [Thread-2005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:49,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:49,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:49,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:49,005 DEBUG [Thread-2016 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:49,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:49,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:49,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:49,006 DEBUG [Thread-2024 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:49,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:49,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:49,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:49,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:49,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:55:49,008 DEBUG [Thread-2013 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:55:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:55:49,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:49,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:49,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:49,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:55:49,009 DEBUG [Thread-2024 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:49,009 DEBUG [Thread-2024 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:49,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:55:49,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:49,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:49,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:49,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:55:49,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:55:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:55:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:55:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:49,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T02:55:49,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:55:49,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:49,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:55:49,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:49,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:55:49,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:49,025 DEBUG [Thread-2020 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:55:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:55:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:55:49,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:49,029 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:55:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-28T02:55:49,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T02:55:49,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2] 2024-11-28T02:55:49,031 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:49,031 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:49,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762609048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732762549039, max timestamp=1732762549051Num uniques 859; 2024-11-28T02:55:49,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/95ced3ebc8b44e2daa21879028f47f44 is 50, key is test_row_0/A:col10/1732762549014/Put/seqid=0 2024-11-28T02:55:49,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T02:55:49,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732762549039, max timestamp=1732762549051Num uniques 859; 2024-11-28T02:55:49,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732762549039, max timestamp=1732762549051Num uniques 859; 2024-11-28T02:55:49,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T02:55:49,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T02:55:49,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742270_1446 (size=14341) 2024-11-28T02:55:49,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/95ced3ebc8b44e2daa21879028f47f44 2024-11-28T02:55:49,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/9724609cf03749ef8e28b78bd1d181ef is 50, key is test_row_0/B:col10/1732762549014/Put/seqid=0 2024-11-28T02:55:49,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762609093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742271_1447 (size=12001) 2024-11-28T02:55:49,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762609094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762609095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762609099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T02:55:49,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762609156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T02:55:49,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:49,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:49,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:49,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:49,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:49,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762609206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762609206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762609206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762609216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T02:55:49,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:49,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:49,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:49,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:49,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T02:55:49,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762609376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762609415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762609426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762609426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762609436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T02:55:49,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:49,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:49,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:49,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:49,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:49,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/9724609cf03749ef8e28b78bd1d181ef 2024-11-28T02:55:49,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/d06e9c0f3e524d309d5e16714b5932da is 50, key is test_row_0/C:col10/1732762549014/Put/seqid=0 2024-11-28T02:55:49,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742272_1448 (size=12001) 2024-11-28T02:55:49,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/d06e9c0f3e524d309d5e16714b5932da 2024-11-28T02:55:49,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/95ced3ebc8b44e2daa21879028f47f44 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/95ced3ebc8b44e2daa21879028f47f44 2024-11-28T02:55:49,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/95ced3ebc8b44e2daa21879028f47f44, entries=200, sequenceid=12, filesize=14.0 K 2024-11-28T02:55:49,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/9724609cf03749ef8e28b78bd1d181ef as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/9724609cf03749ef8e28b78bd1d181ef 2024-11-28T02:55:49,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/9724609cf03749ef8e28b78bd1d181ef, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T02:55:49,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/d06e9c0f3e524d309d5e16714b5932da as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/d06e9c0f3e524d309d5e16714b5932da 2024-11-28T02:55:49,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/d06e9c0f3e524d309d5e16714b5932da, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T02:55:49,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for 2a9027925ba3f01c5e0048c9e37ce432 in 532ms, sequenceid=12, compaction requested=false 2024-11-28T02:55:49,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:49,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T02:55:49,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:49,642 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-28T02:55:49,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:55:49,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:49,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:55:49,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:49,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:55:49,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:49,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/bae16dccdc46408e9bc44b79656b1ef1 is 50, key is test_row_0/A:col10/1732762549094/Put/seqid=0 2024-11-28T02:55:49,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742273_1449 (size=12001) 2024-11-28T02:55:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T02:55:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:49,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:49,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762609730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762609734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762609736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762609736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762609756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762609846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762609846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:49,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:50,049 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/bae16dccdc46408e9bc44b79656b1ef1 2024-11-28T02:55:50,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762610066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:50,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762610066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:50,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/e33fba4a0f034c07a427b5e2200daa21 is 50, key is test_row_0/B:col10/1732762549094/Put/seqid=0 2024-11-28T02:55:50,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742274_1450 (size=12001) 2024-11-28T02:55:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T02:55:50,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:50,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762610256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:50,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762610256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:50,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:50,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762610276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:50,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:50,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762610386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:50,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:50,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762610386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:50,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:50,512 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/e33fba4a0f034c07a427b5e2200daa21 2024-11-28T02:55:50,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/c3c665a583394eb6b9ade358b67e98c6 is 50, key is test_row_0/C:col10/1732762549094/Put/seqid=0 2024-11-28T02:55:50,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742275_1451 (size=12001) 2024-11-28T02:55:50,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762610906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762610906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:50,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:50,921 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/c3c665a583394eb6b9ade358b67e98c6 2024-11-28T02:55:50,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/bae16dccdc46408e9bc44b79656b1ef1 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bae16dccdc46408e9bc44b79656b1ef1 2024-11-28T02:55:50,927 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bae16dccdc46408e9bc44b79656b1ef1, entries=150, sequenceid=47, filesize=11.7 K 2024-11-28T02:55:50,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/e33fba4a0f034c07a427b5e2200daa21 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/e33fba4a0f034c07a427b5e2200daa21 2024-11-28T02:55:50,930 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/e33fba4a0f034c07a427b5e2200daa21, entries=150, sequenceid=47, filesize=11.7 K 2024-11-28T02:55:50,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/c3c665a583394eb6b9ade358b67e98c6 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/c3c665a583394eb6b9ade358b67e98c6 2024-11-28T02:55:50,934 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/c3c665a583394eb6b9ade358b67e98c6, entries=150, sequenceid=47, filesize=11.7 K 2024-11-28T02:55:50,934 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 2a9027925ba3f01c5e0048c9e37ce432 in 1293ms, sequenceid=47, compaction requested=false 2024-11-28T02:55:50,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:50,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:50,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-28T02:55:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-28T02:55:50,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-28T02:55:50,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9050 sec 2024-11-28T02:55:50,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.9100 sec 2024-11-28T02:55:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T02:55:51,176 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:51,176 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-28T02:55:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T02:55:51,178 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:51,178 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:51,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:51,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:55:51,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:55:51,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:51,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:55:51,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:51,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:55:51,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:51,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/f2a4840e145f4748a392cede008a51de is 50, key is test_row_0/A:col10/1732762549733/Put/seqid=0 2024-11-28T02:55:51,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742276_1452 (size=14341) 2024-11-28T02:55:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T02:55:51,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762611318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:51,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762551312, max timestamp=1732762551324Num uniques 859; 2024-11-28T02:55:51,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762551312, max timestamp=1732762551324Num uniques 859; 2024-11-28T02:55:51,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762551312, max timestamp=1732762551324Num uniques 859; 2024-11-28T02:55:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:51,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:51,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:51,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T02:55:51,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:51,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:51,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:51,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762611394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:51,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:51,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762611403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:51,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762611435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:51,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T02:55:51,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:51,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:51,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:51,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T02:55:51,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762611516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:51,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762611516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:51,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T02:55:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:51,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762611645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:51,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/f2a4840e145f4748a392cede008a51de 2024-11-28T02:55:51,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/78276b55e09c496fb2172dcdfdb58b48 is 50, key is test_row_0/B:col10/1732762549733/Put/seqid=0 2024-11-28T02:55:51,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742277_1453 (size=12001) 2024-11-28T02:55:51,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762611736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:51,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:51,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762611736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:51,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:51,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:51,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T02:55:51,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:51,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:51,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:51,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T02:55:51,881 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T02:55:51,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762611926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:51,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762611926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:51,932 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_1' in 2a9027925ba3f01c5e0048c9e37ce432 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:51,932 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_1' in 2a9027925ba3f01c5e0048c9e37ce432 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:51,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T02:55:51,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:51,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:51,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:51,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:51,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762611955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:51,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:51,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762612056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762612056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:52,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:52,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/78276b55e09c496fb2172dcdfdb58b48 2024-11-28T02:55:52,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/ea00c7936ba541e0a7d696d007e7eb7f is 50, key is test_row_0/C:col10/1732762549733/Put/seqid=0 2024-11-28T02:55:52,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T02:55:52,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:52,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:52,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:52,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:52,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:52,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742278_1454 (size=12001) 2024-11-28T02:55:52,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T02:55:52,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:52,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:52,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:52,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:52,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T02:55:52,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T02:55:52,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:52,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:52,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:52,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:52,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:52,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762612466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:52,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/ea00c7936ba541e0a7d696d007e7eb7f 2024-11-28T02:55:52,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/f2a4840e145f4748a392cede008a51de as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/f2a4840e145f4748a392cede008a51de 2024-11-28T02:55:52,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/f2a4840e145f4748a392cede008a51de, entries=200, sequenceid=58, filesize=14.0 K 2024-11-28T02:55:52,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/78276b55e09c496fb2172dcdfdb58b48 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/78276b55e09c496fb2172dcdfdb58b48 2024-11-28T02:55:52,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/78276b55e09c496fb2172dcdfdb58b48, entries=150, sequenceid=58, filesize=11.7 K 2024-11-28T02:55:52,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/ea00c7936ba541e0a7d696d007e7eb7f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ea00c7936ba541e0a7d696d007e7eb7f 2024-11-28T02:55:52,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ea00c7936ba541e0a7d696d007e7eb7f, entries=150, sequenceid=58, filesize=11.7 K 2024-11-28T02:55:52,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2a9027925ba3f01c5e0048c9e37ce432 in 1237ms, sequenceid=58, compaction requested=true 2024-11-28T02:55:52,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:52,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:52,507 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:52,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:52,507 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:52,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:52,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:52,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:52,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:52,507 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:52,507 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/A is initiating minor compaction (all files) 2024-11-28T02:55:52,507 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/A in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:52,508 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/95ced3ebc8b44e2daa21879028f47f44, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bae16dccdc46408e9bc44b79656b1ef1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/f2a4840e145f4748a392cede008a51de] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=39.7 K 2024-11-28T02:55:52,508 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:52,508 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/B is initiating minor compaction (all files) 2024-11-28T02:55:52,508 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95ced3ebc8b44e2daa21879028f47f44, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732762549012 2024-11-28T02:55:52,508 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/B in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:52,508 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/9724609cf03749ef8e28b78bd1d181ef, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/e33fba4a0f034c07a427b5e2200daa21, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/78276b55e09c496fb2172dcdfdb58b48] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=35.2 K 2024-11-28T02:55:52,508 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bae16dccdc46408e9bc44b79656b1ef1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732762549093 2024-11-28T02:55:52,508 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9724609cf03749ef8e28b78bd1d181ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732762549012 2024-11-28T02:55:52,508 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2a4840e145f4748a392cede008a51de, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732762549719 2024-11-28T02:55:52,509 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting e33fba4a0f034c07a427b5e2200daa21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732762549093 2024-11-28T02:55:52,509 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 78276b55e09c496fb2172dcdfdb58b48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732762549719 2024-11-28T02:55:52,518 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:52,519 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:52,519 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/84f2b16dfb764206950c9a21ba1f28ef is 50, key is test_row_0/B:col10/1732762549733/Put/seqid=0 2024-11-28T02:55:52,519 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/c75291c2a7ae4d55895f45a61588b473 is 50, key is test_row_0/A:col10/1732762549733/Put/seqid=0 2024-11-28T02:55:52,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742279_1455 (size=12104) 2024-11-28T02:55:52,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742280_1456 (size=12104) 2024-11-28T02:55:52,537 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/c75291c2a7ae4d55895f45a61588b473 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/c75291c2a7ae4d55895f45a61588b473 2024-11-28T02:55:52,540 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/A of 2a9027925ba3f01c5e0048c9e37ce432 into c75291c2a7ae4d55895f45a61588b473(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:52,541 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:52,541 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/A, priority=13, startTime=1732762552506; duration=0sec 2024-11-28T02:55:52,541 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:52,541 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:A 2024-11-28T02:55:52,541 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:52,542 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:52,542 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/C is initiating minor compaction (all files) 2024-11-28T02:55:52,542 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/C in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:52,542 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/d06e9c0f3e524d309d5e16714b5932da, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/c3c665a583394eb6b9ade358b67e98c6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ea00c7936ba541e0a7d696d007e7eb7f] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=35.2 K 2024-11-28T02:55:52,542 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d06e9c0f3e524d309d5e16714b5932da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732762549012 2024-11-28T02:55:52,542 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3c665a583394eb6b9ade358b67e98c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732762549093 2024-11-28T02:55:52,542 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea00c7936ba541e0a7d696d007e7eb7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732762549719 2024-11-28T02:55:52,550 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#C#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:52,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T02:55:52,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:52,551 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-28T02:55:52,551 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/5e1b789af2f840db95504513ba4b9443 is 50, key is test_row_0/C:col10/1732762549733/Put/seqid=0 2024-11-28T02:55:52,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:55:52,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:52,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:55:52,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:52,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:55:52,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:52,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/21739f595d2347bb854ac0134b62400e is 50, key is test_row_0/A:col10/1732762551361/Put/seqid=0 2024-11-28T02:55:52,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742281_1457 (size=12104) 2024-11-28T02:55:52,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742282_1458 (size=12001) 2024-11-28T02:55:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:52,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:52,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762612611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:52,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762612618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:52,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762612726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762612726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:52,925 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/84f2b16dfb764206950c9a21ba1f28ef as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/84f2b16dfb764206950c9a21ba1f28ef 2024-11-28T02:55:52,928 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/B of 2a9027925ba3f01c5e0048c9e37ce432 into 84f2b16dfb764206950c9a21ba1f28ef(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:52,928 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:52,928 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/B, priority=13, startTime=1732762552507; duration=0sec 2024-11-28T02:55:52,929 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:52,929 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:B 2024-11-28T02:55:52,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762612946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762612946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:52,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:52,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:52,964 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/21739f595d2347bb854ac0134b62400e 2024-11-28T02:55:52,965 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/5e1b789af2f840db95504513ba4b9443 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/5e1b789af2f840db95504513ba4b9443 2024-11-28T02:55:52,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/5b85e13e972d48a78928247d9b957a2f is 50, key is test_row_0/B:col10/1732762551361/Put/seqid=0 2024-11-28T02:55:52,971 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/C of 2a9027925ba3f01c5e0048c9e37ce432 into 5e1b789af2f840db95504513ba4b9443(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:52,971 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:52,971 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/C, priority=13, startTime=1732762552507; duration=0sec 2024-11-28T02:55:52,971 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:52,971 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:C 2024-11-28T02:55:52,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742283_1459 (size=12001) 2024-11-28T02:55:53,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:53,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762613265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:53,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:53,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:53,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762613266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:53,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:53,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:53,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:53,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:53,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T02:55:53,373 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/5b85e13e972d48a78928247d9b957a2f 2024-11-28T02:55:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/3d24ac405103405d814cfbddae97167e is 50, key is test_row_0/C:col10/1732762551361/Put/seqid=0 2024-11-28T02:55:53,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742284_1460 (size=12001) 2024-11-28T02:55:53,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762613476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:53,480 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 2a9027925ba3f01c5e0048c9e37ce432 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:53,784 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/3d24ac405103405d814cfbddae97167e 2024-11-28T02:55:53,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/21739f595d2347bb854ac0134b62400e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/21739f595d2347bb854ac0134b62400e 2024-11-28T02:55:53,791 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/21739f595d2347bb854ac0134b62400e, entries=150, sequenceid=94, filesize=11.7 K 2024-11-28T02:55:53,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/5b85e13e972d48a78928247d9b957a2f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/5b85e13e972d48a78928247d9b957a2f 2024-11-28T02:55:53,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762613786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:53,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762613786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:53,795 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/5b85e13e972d48a78928247d9b957a2f, entries=150, sequenceid=94, filesize=11.7 K 2024-11-28T02:55:53,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/3d24ac405103405d814cfbddae97167e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3d24ac405103405d814cfbddae97167e 2024-11-28T02:55:53,798 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3d24ac405103405d814cfbddae97167e, entries=150, sequenceid=94, filesize=11.7 K 2024-11-28T02:55:53,799 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 2a9027925ba3f01c5e0048c9e37ce432 in 1248ms, sequenceid=94, compaction requested=false 2024-11-28T02:55:53,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:53,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:53,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-28T02:55:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-28T02:55:53,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-28T02:55:53,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6210 sec 2024-11-28T02:55:53,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.6260 sec 2024-11-28T02:55:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:53,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T02:55:53,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:55:53,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:53,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:55:53,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:53,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:55:53,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:53,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/de33733f48554272a0494744e47c824d is 50, key is test_row_0/A:col10/1732762552617/Put/seqid=0 2024-11-28T02:55:53,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742285_1461 (size=16681) 2024-11-28T02:55:54,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762614007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762614013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:54,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762614126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762614126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:54,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762614346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:54,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762614346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:54,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/de33733f48554272a0494744e47c824d 2024-11-28T02:55:54,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/3f605dc0d7a640ca88d2826cd627502a is 50, key is test_row_0/B:col10/1732762552617/Put/seqid=0 2024-11-28T02:55:54,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742286_1462 (size=12001) 2024-11-28T02:55:54,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762614655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762614655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:54,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/3f605dc0d7a640ca88d2826cd627502a 2024-11-28T02:55:54,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/f95c84568b10465ea26d18cd0037f243 is 50, key is test_row_0/C:col10/1732762552617/Put/seqid=0 2024-11-28T02:55:54,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742287_1463 (size=12001) 2024-11-28T02:55:54,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762614806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:54,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762614806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:54,811 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 2a9027925ba3f01c5e0048c9e37ce432 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:54,812 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 2a9027925ba3f01c5e0048c9e37ce432 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:54,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:55,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762615166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:55,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/f95c84568b10465ea26d18cd0037f243 2024-11-28T02:55:55,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762615176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:55,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/de33733f48554272a0494744e47c824d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/de33733f48554272a0494744e47c824d 2024-11-28T02:55:55,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/de33733f48554272a0494744e47c824d, entries=250, sequenceid=109, filesize=16.3 K 2024-11-28T02:55:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/3f605dc0d7a640ca88d2826cd627502a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3f605dc0d7a640ca88d2826cd627502a 2024-11-28T02:55:55,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3f605dc0d7a640ca88d2826cd627502a, entries=150, sequenceid=109, filesize=11.7 K 2024-11-28T02:55:55,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/f95c84568b10465ea26d18cd0037f243 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/f95c84568b10465ea26d18cd0037f243 2024-11-28T02:55:55,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/f95c84568b10465ea26d18cd0037f243, entries=150, sequenceid=109, filesize=11.7 K 2024-11-28T02:55:55,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2a9027925ba3f01c5e0048c9e37ce432 in 1247ms, sequenceid=109, compaction requested=true 2024-11-28T02:55:55,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:55,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:55,197 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:55,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:55,197 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:55,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:55,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:55,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:55,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:55,198 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:55,199 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/A is initiating minor compaction (all files) 2024-11-28T02:55:55,199 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/A in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:55,199 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/c75291c2a7ae4d55895f45a61588b473, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/21739f595d2347bb854ac0134b62400e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/de33733f48554272a0494744e47c824d] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=39.8 K 2024-11-28T02:55:55,199 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:55,199 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c75291c2a7ae4d55895f45a61588b473, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732762549719 2024-11-28T02:55:55,199 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/B is initiating minor compaction (all files) 2024-11-28T02:55:55,199 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/B in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:55,199 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/84f2b16dfb764206950c9a21ba1f28ef, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/5b85e13e972d48a78928247d9b957a2f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3f605dc0d7a640ca88d2826cd627502a] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=35.3 K 2024-11-28T02:55:55,199 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21739f595d2347bb854ac0134b62400e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732762551361 2024-11-28T02:55:55,199 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 84f2b16dfb764206950c9a21ba1f28ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732762549719 2024-11-28T02:55:55,199 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting de33733f48554272a0494744e47c824d, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732762552599 2024-11-28T02:55:55,200 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b85e13e972d48a78928247d9b957a2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732762551361 2024-11-28T02:55:55,200 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f605dc0d7a640ca88d2826cd627502a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732762552610 2024-11-28T02:55:55,205 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#B#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:55,205 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/c4c4cde0c3a74a9dbece855550499050 is 50, key is test_row_0/B:col10/1732762552617/Put/seqid=0 2024-11-28T02:55:55,206 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#A#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:55,206 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/8257ffcd34464e9cbf937033768ac761 is 50, key is test_row_0/A:col10/1732762552617/Put/seqid=0 2024-11-28T02:55:55,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742288_1464 (size=12207) 2024-11-28T02:55:55,215 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/c4c4cde0c3a74a9dbece855550499050 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c4c4cde0c3a74a9dbece855550499050 2024-11-28T02:55:55,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742289_1465 (size=12207) 2024-11-28T02:55:55,222 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/B of 2a9027925ba3f01c5e0048c9e37ce432 into c4c4cde0c3a74a9dbece855550499050(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:55,222 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:55,222 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/B, priority=13, startTime=1732762555197; duration=0sec 2024-11-28T02:55:55,222 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:55,222 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:B 2024-11-28T02:55:55,222 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:55:55,223 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:55:55,223 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/C is initiating minor compaction (all files) 2024-11-28T02:55:55,223 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/C in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:55,224 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/5e1b789af2f840db95504513ba4b9443, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3d24ac405103405d814cfbddae97167e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/f95c84568b10465ea26d18cd0037f243] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=35.3 K 2024-11-28T02:55:55,224 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e1b789af2f840db95504513ba4b9443, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732762549719 2024-11-28T02:55:55,224 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d24ac405103405d814cfbddae97167e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732762551361 2024-11-28T02:55:55,224 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting f95c84568b10465ea26d18cd0037f243, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732762552610 2024-11-28T02:55:55,233 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#C#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:55,234 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/31f22aa7f7c94085b4618789f2047e6f is 50, key is test_row_0/C:col10/1732762552617/Put/seqid=0 2024-11-28T02:55:55,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742290_1466 (size=12207) 2024-11-28T02:55:55,249 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/31f22aa7f7c94085b4618789f2047e6f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/31f22aa7f7c94085b4618789f2047e6f 2024-11-28T02:55:55,253 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/C of 2a9027925ba3f01c5e0048c9e37ce432 into 31f22aa7f7c94085b4618789f2047e6f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:55,253 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:55,253 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/C, priority=13, startTime=1732762555198; duration=0sec 2024-11-28T02:55:55,253 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:55,253 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:C 2024-11-28T02:55:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T02:55:55,346 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:55,346 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-28T02:55:55,347 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T02:55:55,348 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:55,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T02:55:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:55,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T02:55:55,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T02:55:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:55,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:55:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:55,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:55,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:55:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:55,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:55,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:55:55,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:55,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:55,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/184e8860d53a470eb05c35342455e5bd is 50, key is test_row_0/A:col10/1732762555498/Put/seqid=0 2024-11-28T02:55:55,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742291_1467 (size=14391) 2024-11-28T02:55:55,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762615549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:55,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:55,625 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/8257ffcd34464e9cbf937033768ac761 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/8257ffcd34464e9cbf937033768ac761 2024-11-28T02:55:55,629 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/A of 2a9027925ba3f01c5e0048c9e37ce432 into 8257ffcd34464e9cbf937033768ac761(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:55,629 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:55,629 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/A, priority=13, startTime=1732762555197; duration=0sec 2024-11-28T02:55:55,629 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:55,629 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:A 2024-11-28T02:55:55,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T02:55:55,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:55,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:55,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:55,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:55,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T02:55:55,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762615666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:55,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:55,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:55,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:55,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T02:55:55,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:55,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:55,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:55,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:55,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:55,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762615876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:55,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:55,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:55,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:55,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/184e8860d53a470eb05c35342455e5bd 2024-11-28T02:55:55,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/8b2edaf186004893b617dae49131ee73 is 50, key is test_row_0/B:col10/1732762555498/Put/seqid=0 2024-11-28T02:55:55,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742292_1468 (size=12051) 2024-11-28T02:55:55,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/8b2edaf186004893b617dae49131ee73 2024-11-28T02:55:55,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/ad7eac5dbde744ac9ede5257bd9c2eac is 50, key is test_row_0/C:col10/1732762555498/Put/seqid=0 2024-11-28T02:55:55,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742293_1469 (size=12051) 2024-11-28T02:55:55,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/ad7eac5dbde744ac9ede5257bd9c2eac 2024-11-28T02:55:55,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/184e8860d53a470eb05c35342455e5bd as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/184e8860d53a470eb05c35342455e5bd 2024-11-28T02:55:55,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/184e8860d53a470eb05c35342455e5bd, entries=200, sequenceid=131, filesize=14.1 K 2024-11-28T02:55:55,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/8b2edaf186004893b617dae49131ee73 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/8b2edaf186004893b617dae49131ee73 2024-11-28T02:55:55,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/8b2edaf186004893b617dae49131ee73, entries=150, sequenceid=131, filesize=11.8 K 2024-11-28T02:55:55,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/ad7eac5dbde744ac9ede5257bd9c2eac as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ad7eac5dbde744ac9ede5257bd9c2eac 2024-11-28T02:55:55,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ad7eac5dbde744ac9ede5257bd9c2eac, entries=150, sequenceid=131, filesize=11.8 K 2024-11-28T02:55:55,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2a9027925ba3f01c5e0048c9e37ce432 in 449ms, sequenceid=131, compaction requested=false 2024-11-28T02:55:55,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:55,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T02:55:55,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:55,958 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T02:55:55,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:55:55,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:55,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:55:55,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:55,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:55:55,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:55,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/bb6e9f692cf34fa08043ddcad75d9802 is 50, key is test_row_0/A:col10/1732762555545/Put/seqid=0 2024-11-28T02:55:55,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742294_1470 (size=12151) 2024-11-28T02:55:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T02:55:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:56,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:56,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762616248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:56,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762616249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:56,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762616253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:56,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762616356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:56,369 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/bb6e9f692cf34fa08043ddcad75d9802 2024-11-28T02:55:56,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762616366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762616366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:56,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/c6973917ac6b4ec4bbcf7511b3c24af4 is 50, key is test_row_0/B:col10/1732762555545/Put/seqid=0 2024-11-28T02:55:56,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742295_1471 (size=12151) 2024-11-28T02:55:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T02:55:56,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762616566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762616586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762616586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:56,778 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/c6973917ac6b4ec4bbcf7511b3c24af4 2024-11-28T02:55:56,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/25f628f5f2214968a3372f4f032e579a is 50, key is test_row_0/C:col10/1732762555545/Put/seqid=0 2024-11-28T02:55:56,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742296_1472 (size=12151) 2024-11-28T02:55:56,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762616836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762616836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,840 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 2a9027925ba3f01c5e0048c9e37ce432 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:56,840 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 2a9027925ba3f01c5e0048c9e37ce432 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:55:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:56,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762616886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:56,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762616896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762616896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:57,187 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/25f628f5f2214968a3372f4f032e579a 2024-11-28T02:55:57,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/bb6e9f692cf34fa08043ddcad75d9802 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bb6e9f692cf34fa08043ddcad75d9802 2024-11-28T02:55:57,192 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bb6e9f692cf34fa08043ddcad75d9802, entries=150, sequenceid=144, filesize=11.9 K 2024-11-28T02:55:57,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/c6973917ac6b4ec4bbcf7511b3c24af4 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c6973917ac6b4ec4bbcf7511b3c24af4 2024-11-28T02:55:57,196 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c6973917ac6b4ec4bbcf7511b3c24af4, entries=150, sequenceid=144, filesize=11.9 K 2024-11-28T02:55:57,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/25f628f5f2214968a3372f4f032e579a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/25f628f5f2214968a3372f4f032e579a 2024-11-28T02:55:57,199 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/25f628f5f2214968a3372f4f032e579a, entries=150, sequenceid=144, filesize=11.9 K 2024-11-28T02:55:57,200 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2a9027925ba3f01c5e0048c9e37ce432 in 1242ms, sequenceid=144, compaction requested=true 2024-11-28T02:55:57,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:57,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:57,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-28T02:55:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-28T02:55:57,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-28T02:55:57,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8530 sec 2024-11-28T02:55:57,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.8570 sec 2024-11-28T02:55:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:57,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T02:55:57,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762556246, max timestamp=1732762557410Num uniques 859; 2024-11-28T02:55:57,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762556246, max timestamp=1732762557410Num uniques 859; 2024-11-28T02:55:57,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:55:57,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:55:57,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:55:57,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:57,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:55:57,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:57,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:55:57,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:55:57,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/9c5f22afaaab484aa69e2e20ef7f719f is 50, key is test_row_0/A:col10/1732762557410/Put/seqid=0 2024-11-28T02:55:57,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742297_1473 (size=16931) 2024-11-28T02:55:57,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762617468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:57,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:57,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762617469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:57,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762617471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T02:55:57,496 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:57,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-28T02:55:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T02:55:57,498 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:57,498 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:57,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:57,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762617576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:57,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762617586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:57,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762617586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T02:55:57,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T02:55:57,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:57,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:57,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:57,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:57,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:57,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:57,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762617796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:57,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:57,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762617796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:57,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:57,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:57,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762617796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:57,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T02:55:57,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:57,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:57,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:57,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:57,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:57,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:57,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T02:55:57,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/9c5f22afaaab484aa69e2e20ef7f719f 2024-11-28T02:55:57,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/c6c285a499214e2c8f44bd52664ec07c is 50, key is test_row_0/B:col10/1732762557410/Put/seqid=0 2024-11-28T02:55:57,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742298_1474 (size=12151) 2024-11-28T02:55:57,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T02:55:57,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:57,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:57,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:57,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:57,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:58,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T02:55:58,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:58,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:58,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:58,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:58,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:58,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:58,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762618106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:58,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:58,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762618106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:58,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T02:55:58,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:58,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762618115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:58,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/c6c285a499214e2c8f44bd52664ec07c 2024-11-28T02:55:58,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/2da244a0a251445cb23a203d81443c83 is 50, key is test_row_0/C:col10/1732762557410/Put/seqid=0 2024-11-28T02:55:58,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742299_1475 (size=12151) 2024-11-28T02:55:58,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T02:55:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:58,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:58,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T02:55:58,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:58,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:58,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:58,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:58,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:58,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T02:55:58,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:58,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:58,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:58,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:58,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:58,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762618616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:58,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762618616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T02:55:58,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762618636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:58,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/2da244a0a251445cb23a203d81443c83 2024-11-28T02:55:58,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/9c5f22afaaab484aa69e2e20ef7f719f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/9c5f22afaaab484aa69e2e20ef7f719f 2024-11-28T02:55:58,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/9c5f22afaaab484aa69e2e20ef7f719f, entries=250, sequenceid=166, filesize=16.5 K 2024-11-28T02:55:58,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/c6c285a499214e2c8f44bd52664ec07c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c6c285a499214e2c8f44bd52664ec07c 2024-11-28T02:55:58,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c6c285a499214e2c8f44bd52664ec07c, entries=150, sequenceid=166, filesize=11.9 K 2024-11-28T02:55:58,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/2da244a0a251445cb23a203d81443c83 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/2da244a0a251445cb23a203d81443c83 2024-11-28T02:55:58,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/2da244a0a251445cb23a203d81443c83, entries=150, sequenceid=166, filesize=11.9 K 2024-11-28T02:55:58,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 2a9027925ba3f01c5e0048c9e37ce432 in 1249ms, sequenceid=166, compaction requested=true 2024-11-28T02:55:58,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:58,661 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:55:58,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:55:58,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:58,661 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:55:58,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:55:58,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:58,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:55:58,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:58,662 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55680 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:55:58,662 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/A is initiating minor compaction (all files) 2024-11-28T02:55:58,662 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48560 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:55:58,662 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/B is initiating minor compaction (all files) 2024-11-28T02:55:58,662 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/A in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:58,662 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/B in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:58,662 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c4c4cde0c3a74a9dbece855550499050, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/8b2edaf186004893b617dae49131ee73, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c6973917ac6b4ec4bbcf7511b3c24af4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c6c285a499214e2c8f44bd52664ec07c] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=47.4 K 2024-11-28T02:55:58,662 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/8257ffcd34464e9cbf937033768ac761, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/184e8860d53a470eb05c35342455e5bd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bb6e9f692cf34fa08043ddcad75d9802, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/9c5f22afaaab484aa69e2e20ef7f719f] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=54.4 K 2024-11-28T02:55:58,663 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting c4c4cde0c3a74a9dbece855550499050, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732762552610 2024-11-28T02:55:58,663 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8257ffcd34464e9cbf937033768ac761, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732762552610 2024-11-28T02:55:58,663 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b2edaf186004893b617dae49131ee73, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732762554006 2024-11-28T02:55:58,663 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 184e8860d53a470eb05c35342455e5bd, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732762554000 2024-11-28T02:55:58,663 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting c6973917ac6b4ec4bbcf7511b3c24af4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732762555537 2024-11-28T02:55:58,663 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb6e9f692cf34fa08043ddcad75d9802, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732762555537 2024-11-28T02:55:58,663 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting c6c285a499214e2c8f44bd52664ec07c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732762556252 2024-11-28T02:55:58,663 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c5f22afaaab484aa69e2e20ef7f719f, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732762556246 2024-11-28T02:55:58,670 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#B#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:58,671 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/cbef5792c243497598eed9c769a7c0b8 is 50, key is test_row_0/B:col10/1732762557410/Put/seqid=0 2024-11-28T02:55:58,672 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#A#compaction#397 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:58,672 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/1cc276baf43a4de1bf047fa71b39d337 is 50, key is test_row_0/A:col10/1732762557410/Put/seqid=0 2024-11-28T02:55:58,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742300_1476 (size=12493) 2024-11-28T02:55:58,678 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/cbef5792c243497598eed9c769a7c0b8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/cbef5792c243497598eed9c769a7c0b8 2024-11-28T02:55:58,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742301_1477 (size=12493) 2024-11-28T02:55:58,682 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/B of 2a9027925ba3f01c5e0048c9e37ce432 into cbef5792c243497598eed9c769a7c0b8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:58,682 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:58,682 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/B, priority=12, startTime=1732762558661; duration=0sec 2024-11-28T02:55:58,682 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:55:58,682 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:B 2024-11-28T02:55:58,682 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:55:58,683 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48560 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:55:58,683 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/C is initiating minor compaction (all files) 2024-11-28T02:55:58,683 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/C in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:58,684 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/31f22aa7f7c94085b4618789f2047e6f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ad7eac5dbde744ac9ede5257bd9c2eac, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/25f628f5f2214968a3372f4f032e579a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/2da244a0a251445cb23a203d81443c83] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=47.4 K 2024-11-28T02:55:58,684 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 31f22aa7f7c94085b4618789f2047e6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732762552610 2024-11-28T02:55:58,684 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting ad7eac5dbde744ac9ede5257bd9c2eac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732762554006 2024-11-28T02:55:58,684 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 25f628f5f2214968a3372f4f032e579a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732762555537 2024-11-28T02:55:58,684 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/1cc276baf43a4de1bf047fa71b39d337 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/1cc276baf43a4de1bf047fa71b39d337 2024-11-28T02:55:58,684 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2da244a0a251445cb23a203d81443c83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732762556252 2024-11-28T02:55:58,688 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/A of 2a9027925ba3f01c5e0048c9e37ce432 into 1cc276baf43a4de1bf047fa71b39d337(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:58,689 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:58,689 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/A, priority=12, startTime=1732762558661; duration=0sec 2024-11-28T02:55:58,689 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:58,689 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:A 2024-11-28T02:55:58,693 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#C#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:55:58,693 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/6ca323bc464044959e317bf9002accde is 50, key is test_row_0/C:col10/1732762557410/Put/seqid=0 2024-11-28T02:55:58,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742302_1478 (size=12493) 2024-11-28T02:55:58,703 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/6ca323bc464044959e317bf9002accde as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/6ca323bc464044959e317bf9002accde 2024-11-28T02:55:58,712 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/C of 2a9027925ba3f01c5e0048c9e37ce432 into 6ca323bc464044959e317bf9002accde(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:55:58,712 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:58,712 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/C, priority=12, startTime=1732762558662; duration=0sec 2024-11-28T02:55:58,712 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:55:58,713 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:C 2024-11-28T02:55:58,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T02:55:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:58,721 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T02:55:58,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:55:58,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:58,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:55:58,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:58,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:55:58,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:58,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/5c5ffeca43524355a03711d461bd0356 is 50, key is test_row_0/A:col10/1732762557458/Put/seqid=0 2024-11-28T02:55:58,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742303_1479 (size=12151) 2024-11-28T02:55:58,753 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/5c5ffeca43524355a03711d461bd0356 2024-11-28T02:55:58,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/3411c7f1c6e745ea9ce324b5ca9a01e0 is 50, key is test_row_0/B:col10/1732762557458/Put/seqid=0 2024-11-28T02:55:58,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742304_1480 (size=12151) 2024-11-28T02:55:58,790 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/3411c7f1c6e745ea9ce324b5ca9a01e0 2024-11-28T02:55:58,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/03ba03fc8eaa4b629e047df0a30d1a65 is 50, key is test_row_0/C:col10/1732762557458/Put/seqid=0 2024-11-28T02:55:58,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742305_1481 (size=12151) 2024-11-28T02:55:58,811 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/03ba03fc8eaa4b629e047df0a30d1a65 2024-11-28T02:55:58,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/5c5ffeca43524355a03711d461bd0356 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/5c5ffeca43524355a03711d461bd0356 2024-11-28T02:55:58,820 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/5c5ffeca43524355a03711d461bd0356, entries=150, sequenceid=189, filesize=11.9 K 2024-11-28T02:55:58,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/3411c7f1c6e745ea9ce324b5ca9a01e0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3411c7f1c6e745ea9ce324b5ca9a01e0 2024-11-28T02:55:58,823 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3411c7f1c6e745ea9ce324b5ca9a01e0, entries=150, sequenceid=189, filesize=11.9 K 2024-11-28T02:55:58,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/03ba03fc8eaa4b629e047df0a30d1a65 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/03ba03fc8eaa4b629e047df0a30d1a65 2024-11-28T02:55:58,827 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/03ba03fc8eaa4b629e047df0a30d1a65, entries=150, sequenceid=189, filesize=11.9 K 2024-11-28T02:55:58,827 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 2a9027925ba3f01c5e0048c9e37ce432 in 107ms, sequenceid=189, compaction requested=false 2024-11-28T02:55:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:55:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-28T02:55:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-28T02:55:58,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-28T02:55:58,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3310 sec 2024-11-28T02:55:58,834 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.3360 sec 2024-11-28T02:55:59,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:55:59,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T02:55:59,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T02:55:59,646 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:55:59,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:55:59,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:59,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:55:59,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:59,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:55:59,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:55:59,646 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:55:59,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-28T02:55:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T02:55:59,649 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:55:59,650 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:55:59,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:55:59,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/92cf7242ef9e4089ac389603294bbd40 is 50, key is test_row_0/A:col10/1732762559640/Put/seqid=0 2024-11-28T02:55:59,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742306_1482 (size=16931) 2024-11-28T02:55:59,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762559692, max timestamp=1732762559705Num uniques 859; 2024-11-28T02:55:59,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762559692, max timestamp=1732762559705Num uniques 859; 2024-11-28T02:55:59,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762559692, max timestamp=1732762559705Num uniques 859; 2024-11-28T02:55:59,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:59,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:59,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:59,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:55:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762619706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:59,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:59,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:59,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:59,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T02:55:59,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762619771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:59,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762619778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:59,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T02:55:59,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:59,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:59,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:59,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:59,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:59,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762619816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:59,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762619886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:59,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:59,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:55:59,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:59,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762619886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:55:59,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:59,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:55:59,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:55:59,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T02:55:59,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:59,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:55:59,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:55:59,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:59,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:55:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T02:56:00,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762620036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:00,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:00,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/92cf7242ef9e4089ac389603294bbd40 2024-11-28T02:56:00,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T02:56:00,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:00,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:00,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:00,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:00,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:00,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762620106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:00,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762620106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:00,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/c5c1c1ea69594b7581e0a453f4ead424 is 50, key is test_row_0/B:col10/1732762559640/Put/seqid=0 2024-11-28T02:56:00,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742307_1483 (size=12151) 2024-11-28T02:56:00,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T02:56:00,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:00,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:00,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:00,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:00,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T02:56:00,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762620356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:00,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T02:56:00,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:00,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:00,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:00,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:00,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:00,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762620426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:00,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762620426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:00,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/c5c1c1ea69594b7581e0a453f4ead424 2024-11-28T02:56:00,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/2d47307640624ac3a1f7cda78b9fc886 is 50, key is test_row_0/C:col10/1732762559640/Put/seqid=0 2024-11-28T02:56:00,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742308_1484 (size=12151) 2024-11-28T02:56:00,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/2d47307640624ac3a1f7cda78b9fc886 2024-11-28T02:56:00,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/92cf7242ef9e4089ac389603294bbd40 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/92cf7242ef9e4089ac389603294bbd40 2024-11-28T02:56:00,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/92cf7242ef9e4089ac389603294bbd40, entries=250, sequenceid=199, filesize=16.5 K 2024-11-28T02:56:00,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/c5c1c1ea69594b7581e0a453f4ead424 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c5c1c1ea69594b7581e0a453f4ead424 2024-11-28T02:56:00,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c5c1c1ea69594b7581e0a453f4ead424, entries=150, sequenceid=199, filesize=11.9 K 2024-11-28T02:56:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/2d47307640624ac3a1f7cda78b9fc886 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/2d47307640624ac3a1f7cda78b9fc886 2024-11-28T02:56:00,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/2d47307640624ac3a1f7cda78b9fc886, entries=150, sequenceid=199, filesize=11.9 K 2024-11-28T02:56:00,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2a9027925ba3f01c5e0048c9e37ce432 in 897ms, sequenceid=199, compaction requested=true 2024-11-28T02:56:00,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:00,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:56:00,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:00,543 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:00,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:56:00,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:00,543 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:00,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:56:00,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:00,544 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:00,544 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:00,544 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/B is initiating minor compaction (all files) 2024-11-28T02:56:00,544 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/A is initiating minor compaction (all files) 2024-11-28T02:56:00,544 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/B in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:00,544 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/A in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:00,544 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/cbef5792c243497598eed9c769a7c0b8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3411c7f1c6e745ea9ce324b5ca9a01e0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c5c1c1ea69594b7581e0a453f4ead424] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=35.9 K 2024-11-28T02:56:00,544 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/1cc276baf43a4de1bf047fa71b39d337, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/5c5ffeca43524355a03711d461bd0356, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/92cf7242ef9e4089ac389603294bbd40] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=40.6 K 2024-11-28T02:56:00,544 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting cbef5792c243497598eed9c769a7c0b8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732762556252 2024-11-28T02:56:00,544 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cc276baf43a4de1bf047fa71b39d337, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732762556252 2024-11-28T02:56:00,545 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3411c7f1c6e745ea9ce324b5ca9a01e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732762557458 2024-11-28T02:56:00,545 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c5ffeca43524355a03711d461bd0356, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732762557458 2024-11-28T02:56:00,545 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting c5c1c1ea69594b7581e0a453f4ead424, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732762559640 2024-11-28T02:56:00,545 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92cf7242ef9e4089ac389603294bbd40, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732762559639 2024-11-28T02:56:00,550 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#A#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:00,551 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/ff23f74dba3249468227bf027283ed25 is 50, key is test_row_0/A:col10/1732762559640/Put/seqid=0 2024-11-28T02:56:00,551 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#B#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:00,551 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/922bf86c9ae246f0bec6393f5ba39272 is 50, key is test_row_0/B:col10/1732762559640/Put/seqid=0 2024-11-28T02:56:00,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742310_1486 (size=12595) 2024-11-28T02:56:00,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742309_1485 (size=12595) 2024-11-28T02:56:00,566 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/ff23f74dba3249468227bf027283ed25 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/ff23f74dba3249468227bf027283ed25 2024-11-28T02:56:00,569 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/A of 2a9027925ba3f01c5e0048c9e37ce432 into ff23f74dba3249468227bf027283ed25(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:00,569 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:00,569 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/A, priority=13, startTime=1732762560543; duration=0sec 2024-11-28T02:56:00,570 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:00,570 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:A 2024-11-28T02:56:00,570 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:00,570 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:00,570 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/C is initiating minor compaction (all files) 2024-11-28T02:56:00,570 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/C in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:00,570 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/6ca323bc464044959e317bf9002accde, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/03ba03fc8eaa4b629e047df0a30d1a65, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/2d47307640624ac3a1f7cda78b9fc886] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=35.9 K 2024-11-28T02:56:00,571 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ca323bc464044959e317bf9002accde, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732762556252 2024-11-28T02:56:00,571 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03ba03fc8eaa4b629e047df0a30d1a65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732762557458 2024-11-28T02:56:00,571 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d47307640624ac3a1f7cda78b9fc886, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732762559640 2024-11-28T02:56:00,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T02:56:00,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:00,571 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T02:56:00,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:56:00,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:00,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:56:00,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:00,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:56:00,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:00,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/b9a6c76d0d91490195519adc824ce91e is 50, key is test_row_0/A:col10/1732762559777/Put/seqid=0 2024-11-28T02:56:00,580 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#C#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:00,581 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/f37983b644d84893b3da39e0bb7912de is 50, key is test_row_0/C:col10/1732762559640/Put/seqid=0 2024-11-28T02:56:00,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742312_1488 (size=12595) 2024-11-28T02:56:00,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742311_1487 (size=12151) 2024-11-28T02:56:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T02:56:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:56:00,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:00,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762620886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:00,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:00,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762620888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:00,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762620892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:00,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762620946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762620946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:00,967 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/922bf86c9ae246f0bec6393f5ba39272 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/922bf86c9ae246f0bec6393f5ba39272 2024-11-28T02:56:00,970 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/B of 2a9027925ba3f01c5e0048c9e37ce432 into 922bf86c9ae246f0bec6393f5ba39272(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:00,970 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:00,970 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/B, priority=13, startTime=1732762560543; duration=0sec 2024-11-28T02:56:00,970 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:00,970 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:B 2024-11-28T02:56:00,988 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/b9a6c76d0d91490195519adc824ce91e 2024-11-28T02:56:00,988 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/f37983b644d84893b3da39e0bb7912de as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/f37983b644d84893b3da39e0bb7912de 2024-11-28T02:56:00,992 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/C of 2a9027925ba3f01c5e0048c9e37ce432 into f37983b644d84893b3da39e0bb7912de(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:00,992 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:00,992 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/C, priority=13, startTime=1732762560543; duration=0sec 2024-11-28T02:56:00,992 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:00,992 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:C 2024-11-28T02:56:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/21d7893aeaca496f8c639eaa21fa3387 is 50, key is test_row_0/B:col10/1732762559777/Put/seqid=0 2024-11-28T02:56:00,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742313_1489 (size=12151) 2024-11-28T02:56:01,000 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/21d7893aeaca496f8c639eaa21fa3387 2024-11-28T02:56:01,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762620996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/dd8aefbb7154409dbef0f646a5799abc is 50, key is test_row_0/C:col10/1732762559777/Put/seqid=0 2024-11-28T02:56:01,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742314_1490 (size=12151) 2024-11-28T02:56:01,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762621006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762621006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762621206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762621216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762621226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,408 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/dd8aefbb7154409dbef0f646a5799abc 2024-11-28T02:56:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/b9a6c76d0d91490195519adc824ce91e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/b9a6c76d0d91490195519adc824ce91e 2024-11-28T02:56:01,414 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/b9a6c76d0d91490195519adc824ce91e, entries=150, sequenceid=236, filesize=11.9 K 2024-11-28T02:56:01,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/21d7893aeaca496f8c639eaa21fa3387 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/21d7893aeaca496f8c639eaa21fa3387 2024-11-28T02:56:01,417 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/21d7893aeaca496f8c639eaa21fa3387, entries=150, sequenceid=236, filesize=11.9 K 2024-11-28T02:56:01,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/dd8aefbb7154409dbef0f646a5799abc as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/dd8aefbb7154409dbef0f646a5799abc 2024-11-28T02:56:01,420 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/dd8aefbb7154409dbef0f646a5799abc, entries=150, sequenceid=236, filesize=11.9 K 2024-11-28T02:56:01,421 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 2a9027925ba3f01c5e0048c9e37ce432 in 850ms, sequenceid=236, compaction requested=false 2024-11-28T02:56:01,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:01,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:01,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-28T02:56:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-28T02:56:01,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-28T02:56:01,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7720 sec 2024-11-28T02:56:01,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.7810 sec 2024-11-28T02:56:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:56:01,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:56:01,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:56:01,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:01,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:56:01,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:01,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:56:01,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:01,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/c66f91b89d164a3f8606fab3d560f1df is 50, key is test_row_0/A:col10/1732762560884/Put/seqid=0 2024-11-28T02:56:01,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742315_1491 (size=12147) 2024-11-28T02:56:01,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/c66f91b89d164a3f8606fab3d560f1df 2024-11-28T02:56:01,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/437f2e01f91a46f0863c6d230466214a is 50, key is test_row_0/B:col10/1732762560884/Put/seqid=0 2024-11-28T02:56:01,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742316_1492 (size=9757) 2024-11-28T02:56:01,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762621581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762621581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732762561547, max timestamp=1732762561587Num uniques 859; 2024-11-28T02:56:01,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T02:56:01,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762561566, max timestamp=1732762561587Num uniques 859; 2024-11-28T02:56:01,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762561566, max timestamp=1732762561587Num uniques 859; 2024-11-28T02:56:01,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:56:01,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:56:01,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762621707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762621708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762621710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T02:56:01,806 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:01,807 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:56:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-28T02:56:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T02:56:01,808 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:56:01,809 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:56:01,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:56:01,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762621825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762621826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762621836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T02:56:01,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/437f2e01f91a46f0863c6d230466214a 2024-11-28T02:56:01,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/3b2dd2033b244c4f8816021de9dcebab is 50, key is test_row_0/C:col10/1732762560884/Put/seqid=0 2024-11-28T02:56:01,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T02:56:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:01,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:01,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742317_1493 (size=9757) 2024-11-28T02:56:01,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/3b2dd2033b244c4f8816021de9dcebab 2024-11-28T02:56:01,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/c66f91b89d164a3f8606fab3d560f1df as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/c66f91b89d164a3f8606fab3d560f1df 2024-11-28T02:56:01,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762621956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,968 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 2a9027925ba3f01c5e0048c9e37ce432 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:56:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/c66f91b89d164a3f8606fab3d560f1df, entries=150, sequenceid=249, filesize=11.9 K 2024-11-28T02:56:01,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/437f2e01f91a46f0863c6d230466214a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/437f2e01f91a46f0863c6d230466214a 2024-11-28T02:56:01,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762621966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:01,973 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 2a9027925ba3f01c5e0048c9e37ce432 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:56:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:01,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/437f2e01f91a46f0863c6d230466214a, entries=100, sequenceid=249, filesize=9.5 K 2024-11-28T02:56:01,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/3b2dd2033b244c4f8816021de9dcebab as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3b2dd2033b244c4f8816021de9dcebab 2024-11-28T02:56:01,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3b2dd2033b244c4f8816021de9dcebab, entries=100, sequenceid=249, filesize=9.5 K 2024-11-28T02:56:01,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 2a9027925ba3f01c5e0048c9e37ce432 in 454ms, sequenceid=249, compaction requested=true 2024-11-28T02:56:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:56:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:01,983 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:01,983 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:01,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:56:01,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:01,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:56:01,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:01,984 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36893 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:01,984 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:01,984 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/A is initiating minor compaction (all files) 2024-11-28T02:56:01,984 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/B is initiating minor compaction (all files) 2024-11-28T02:56:01,984 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/B in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:01,984 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/A in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:01,984 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/922bf86c9ae246f0bec6393f5ba39272, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/21d7893aeaca496f8c639eaa21fa3387, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/437f2e01f91a46f0863c6d230466214a] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=33.7 K 2024-11-28T02:56:01,984 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/ff23f74dba3249468227bf027283ed25, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/b9a6c76d0d91490195519adc824ce91e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/c66f91b89d164a3f8606fab3d560f1df] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=36.0 K 2024-11-28T02:56:01,984 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 922bf86c9ae246f0bec6393f5ba39272, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732762559640 2024-11-28T02:56:01,984 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff23f74dba3249468227bf027283ed25, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732762559640 2024-11-28T02:56:01,984 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d7893aeaca496f8c639eaa21fa3387, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732762559756 2024-11-28T02:56:01,985 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9a6c76d0d91490195519adc824ce91e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732762559756 2024-11-28T02:56:01,985 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 437f2e01f91a46f0863c6d230466214a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732762560884 2024-11-28T02:56:01,985 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c66f91b89d164a3f8606fab3d560f1df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732762560884 2024-11-28T02:56:01,990 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:01,990 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/ccd3dc4b0dd84ca6b66eb01a6b4b008c is 50, key is test_row_0/B:col10/1732762560884/Put/seqid=0 2024-11-28T02:56:01,992 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#A#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:01,993 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/4e856fb765eb45faaa45d9119637b0e8 is 50, key is test_row_0/A:col10/1732762560884/Put/seqid=0 2024-11-28T02:56:01,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742318_1494 (size=12697) 2024-11-28T02:56:02,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742319_1495 (size=12697) 2024-11-28T02:56:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:56:02,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-11-28T02:56:02,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:56:02,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:02,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:56:02,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:02,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:56:02,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:02,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/8fe6433478bd4165a6813f96b6875245 is 50, key is test_row_0/A:col10/1732762561705/Put/seqid=0 2024-11-28T02:56:02,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742320_1496 (size=14741) 2024-11-28T02:56:02,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762622054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762622061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762622062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T02:56:02,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:02,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:02,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:02,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:02,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T02:56:02,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762622176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762622176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762622176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T02:56:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:02,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:02,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:02,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:02,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:02,402 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/ccd3dc4b0dd84ca6b66eb01a6b4b008c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ccd3dc4b0dd84ca6b66eb01a6b4b008c 2024-11-28T02:56:02,405 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/B of 2a9027925ba3f01c5e0048c9e37ce432 into ccd3dc4b0dd84ca6b66eb01a6b4b008c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:02,405 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:02,405 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/B, priority=13, startTime=1732762561983; duration=0sec 2024-11-28T02:56:02,405 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:02,405 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:B 2024-11-28T02:56:02,405 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:02,406 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:02,406 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/C is initiating minor compaction (all files) 2024-11-28T02:56:02,406 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/C in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:02,406 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/f37983b644d84893b3da39e0bb7912de, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/dd8aefbb7154409dbef0f646a5799abc, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3b2dd2033b244c4f8816021de9dcebab] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=33.7 K 2024-11-28T02:56:02,406 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting f37983b644d84893b3da39e0bb7912de, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732762559640 2024-11-28T02:56:02,407 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting dd8aefbb7154409dbef0f646a5799abc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732762559756 2024-11-28T02:56:02,407 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b2dd2033b244c4f8816021de9dcebab, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732762560884 2024-11-28T02:56:02,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762622398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762622398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,408 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/4e856fb765eb45faaa45d9119637b0e8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/4e856fb765eb45faaa45d9119637b0e8 2024-11-28T02:56:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762622398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,412 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/A of 2a9027925ba3f01c5e0048c9e37ce432 into 4e856fb765eb45faaa45d9119637b0e8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:02,412 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:02,412 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/A, priority=13, startTime=1732762561982; duration=0sec 2024-11-28T02:56:02,412 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:02,412 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:A 2024-11-28T02:56:02,414 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#C#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:02,414 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/ace8d694606e4534ac8226a3a9059d14 is 50, key is test_row_0/C:col10/1732762560884/Put/seqid=0 2024-11-28T02:56:02,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742321_1497 (size=12697) 2024-11-28T02:56:02,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T02:56:02,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:02,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:02,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:02,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:02,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:02,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T02:56:02,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/8fe6433478bd4165a6813f96b6875245 2024-11-28T02:56:02,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/e2baf4916da6478bbae06984e0ddf6e5 is 50, key is test_row_0/B:col10/1732762561705/Put/seqid=0 2024-11-28T02:56:02,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742322_1498 (size=12301) 2024-11-28T02:56:02,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/e2baf4916da6478bbae06984e0ddf6e5 2024-11-28T02:56:02,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/9c86614e4c23484eb13a6dfb80c81fc4 is 50, key is test_row_0/C:col10/1732762561705/Put/seqid=0 2024-11-28T02:56:02,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742323_1499 (size=12301) 2024-11-28T02:56:02,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/9c86614e4c23484eb13a6dfb80c81fc4 2024-11-28T02:56:02,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/8fe6433478bd4165a6813f96b6875245 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/8fe6433478bd4165a6813f96b6875245 2024-11-28T02:56:02,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/8fe6433478bd4165a6813f96b6875245, entries=200, sequenceid=286, filesize=14.4 K 2024-11-28T02:56:02,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/e2baf4916da6478bbae06984e0ddf6e5 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/e2baf4916da6478bbae06984e0ddf6e5 2024-11-28T02:56:02,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/e2baf4916da6478bbae06984e0ddf6e5, entries=150, sequenceid=286, filesize=12.0 K 2024-11-28T02:56:02,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/9c86614e4c23484eb13a6dfb80c81fc4 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/9c86614e4c23484eb13a6dfb80c81fc4 2024-11-28T02:56:02,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/9c86614e4c23484eb13a6dfb80c81fc4, entries=150, sequenceid=286, filesize=12.0 K 2024-11-28T02:56:02,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=46.96 KB/48090 for 2a9027925ba3f01c5e0048c9e37ce432 in 451ms, sequenceid=286, compaction requested=false 2024-11-28T02:56:02,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:02,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T02:56:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:02,572 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T02:56:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:56:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:56:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:56:02,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:02,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/761cc95693754e7ca5855fce38ab1c65 is 50, key is test_row_0/A:col10/1732762562060/Put/seqid=0 2024-11-28T02:56:02,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742324_1500 (size=9857) 2024-11-28T02:56:02,580 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/761cc95693754e7ca5855fce38ab1c65 2024-11-28T02:56:02,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/db97bcea32334ed8b19c969fdc8ad0d2 is 50, key is test_row_0/B:col10/1732762562060/Put/seqid=0 2024-11-28T02:56:02,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742325_1501 (size=9857) 2024-11-28T02:56:02,599 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/db97bcea32334ed8b19c969fdc8ad0d2 2024-11-28T02:56:02,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/db13eff90f1446959489d3cf0b36de12 is 50, key is test_row_0/C:col10/1732762562060/Put/seqid=0 2024-11-28T02:56:02,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742326_1502 (size=9857) 2024-11-28T02:56:02,612 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/db13eff90f1446959489d3cf0b36de12 2024-11-28T02:56:02,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/761cc95693754e7ca5855fce38ab1c65 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/761cc95693754e7ca5855fce38ab1c65 2024-11-28T02:56:02,621 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/761cc95693754e7ca5855fce38ab1c65, entries=100, sequenceid=298, filesize=9.6 K 2024-11-28T02:56:02,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/db97bcea32334ed8b19c969fdc8ad0d2 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/db97bcea32334ed8b19c969fdc8ad0d2 2024-11-28T02:56:02,627 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/db97bcea32334ed8b19c969fdc8ad0d2, entries=100, sequenceid=298, filesize=9.6 K 2024-11-28T02:56:02,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/db13eff90f1446959489d3cf0b36de12 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/db13eff90f1446959489d3cf0b36de12 2024-11-28T02:56:02,631 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/db13eff90f1446959489d3cf0b36de12, entries=100, sequenceid=298, filesize=9.6 K 2024-11-28T02:56:02,632 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 2a9027925ba3f01c5e0048c9e37ce432 in 59ms, sequenceid=298, compaction requested=true 2024-11-28T02:56:02,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:02,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:02,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-28T02:56:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-28T02:56:02,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-28T02:56:02,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 824 msec 2024-11-28T02:56:02,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 830 msec 2024-11-28T02:56:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:56:02,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T02:56:02,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:56:02,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:02,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:56:02,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:02,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:56:02,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:02,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/bb99d09d58f74989b8f55f7c0c29e398 is 50, key is test_row_0/A:col10/1732762562732/Put/seqid=0 2024-11-28T02:56:02,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742327_1503 (size=19621) 2024-11-28T02:56:02,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732762562766, max timestamp=1732762562794Num uniques 859; 2024-11-28T02:56:02,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762562779, max timestamp=1732762562794Num uniques 859; 2024-11-28T02:56:02,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732762562766, max timestamp=1732762562794Num uniques 859; 2024-11-28T02:56:02,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:56:02,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T02:56:02,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T02:56:02,821 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/ace8d694606e4534ac8226a3a9059d14 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ace8d694606e4534ac8226a3a9059d14 2024-11-28T02:56:02,826 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/C of 2a9027925ba3f01c5e0048c9e37ce432 into ace8d694606e4534ac8226a3a9059d14(size=12.4 K), total size for store is 34.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:02,826 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:02,826 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/C, priority=13, startTime=1732762561983; duration=0sec 2024-11-28T02:56:02,826 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:02,826 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:C 2024-11-28T02:56:02,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762622850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762622853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762622854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T02:56:02,946 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:02,947 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:56:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-28T02:56:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T02:56:02,948 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:56:02,949 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:56:02,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:56:02,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762622956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762622975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762622976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T02:56:03,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T02:56:03,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:03,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/bb99d09d58f74989b8f55f7c0c29e398 2024-11-28T02:56:03,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/3edf6940d637484dbc16f544f40ba672 is 50, key is test_row_0/B:col10/1732762562732/Put/seqid=0 2024-11-28T02:56:03,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742328_1504 (size=12301) 2024-11-28T02:56:03,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762623175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:03,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:03,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762623196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:03,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762623196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:03,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T02:56:03,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:03,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T02:56:03,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T02:56:03,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:03,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762623496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:03,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762623515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:03,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762623515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:03,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/3edf6940d637484dbc16f544f40ba672 2024-11-28T02:56:03,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/20b27971181c4fa2afeba0faa2818e10 is 50, key is test_row_0/C:col10/1732762562732/Put/seqid=0 2024-11-28T02:56:03,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T02:56:03,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:03,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742329_1505 (size=12301) 2024-11-28T02:56:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T02:56:03,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T02:56:03,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:03,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T02:56:03,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:03,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:03,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/20b27971181c4fa2afeba0faa2818e10 2024-11-28T02:56:03,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/bb99d09d58f74989b8f55f7c0c29e398 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bb99d09d58f74989b8f55f7c0c29e398 2024-11-28T02:56:03,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bb99d09d58f74989b8f55f7c0c29e398, entries=300, sequenceid=308, filesize=19.2 K 2024-11-28T02:56:03,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/3edf6940d637484dbc16f544f40ba672 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3edf6940d637484dbc16f544f40ba672 2024-11-28T02:56:03,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3edf6940d637484dbc16f544f40ba672, entries=150, sequenceid=308, filesize=12.0 K 2024-11-28T02:56:03,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/20b27971181c4fa2afeba0faa2818e10 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/20b27971181c4fa2afeba0faa2818e10 2024-11-28T02:56:03,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/20b27971181c4fa2afeba0faa2818e10, entries=150, sequenceid=308, filesize=12.0 K 2024-11-28T02:56:03,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2a9027925ba3f01c5e0048c9e37ce432 in 1240ms, sequenceid=308, compaction requested=true 2024-11-28T02:56:03,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:03,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:56:03,975 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:56:03,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:03,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:56:03,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:03,975 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:56:03,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:56:03,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:03,975 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56916 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:56:03,975 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47156 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:56:03,975 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/B is initiating minor compaction (all files) 2024-11-28T02:56:03,975 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/A is initiating minor compaction (all files) 2024-11-28T02:56:03,976 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/B in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,976 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/A in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:03,976 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ccd3dc4b0dd84ca6b66eb01a6b4b008c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/e2baf4916da6478bbae06984e0ddf6e5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/db97bcea32334ed8b19c969fdc8ad0d2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3edf6940d637484dbc16f544f40ba672] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=46.1 K 2024-11-28T02:56:03,976 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/4e856fb765eb45faaa45d9119637b0e8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/8fe6433478bd4165a6813f96b6875245, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/761cc95693754e7ca5855fce38ab1c65, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bb99d09d58f74989b8f55f7c0c29e398] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=55.6 K 2024-11-28T02:56:03,979 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting ccd3dc4b0dd84ca6b66eb01a6b4b008c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732762559756 2024-11-28T02:56:03,979 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e856fb765eb45faaa45d9119637b0e8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732762559756 2024-11-28T02:56:03,979 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fe6433478bd4165a6813f96b6875245, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732762561669 2024-11-28T02:56:03,979 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting e2baf4916da6478bbae06984e0ddf6e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732762561705 2024-11-28T02:56:03,979 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 761cc95693754e7ca5855fce38ab1c65, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732762562060 2024-11-28T02:56:03,979 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting db97bcea32334ed8b19c969fdc8ad0d2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732762562060 2024-11-28T02:56:03,979 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb99d09d58f74989b8f55f7c0c29e398, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732762562721 2024-11-28T02:56:03,979 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3edf6940d637484dbc16f544f40ba672, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732762562728 2024-11-28T02:56:03,987 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#A#compaction#426 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:03,987 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#B#compaction#427 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:03,987 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/3415c5a2f48e4456ae57b4a4a03ea9ab is 50, key is test_row_0/A:col10/1732762562732/Put/seqid=0 2024-11-28T02:56:03,988 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/f967db3fc9ef4063b26f73440d1b5009 is 50, key is test_row_0/B:col10/1732762562732/Put/seqid=0 2024-11-28T02:56:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:56:03,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-28T02:56:03,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:56:03,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:03,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:56:03,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:03,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:56:03,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:03,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742330_1506 (size=12983) 2024-11-28T02:56:03,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742331_1507 (size=12983) 2024-11-28T02:56:03,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/0bf394d3e85c4cf88c821fdc04c82da8 is 50, key is test_row_0/A:col10/1732762563992/Put/seqid=0 2024-11-28T02:56:04,001 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/3415c5a2f48e4456ae57b4a4a03ea9ab as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/3415c5a2f48e4456ae57b4a4a03ea9ab 2024-11-28T02:56:04,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742332_1508 (size=14741) 2024-11-28T02:56:04,005 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/A of 2a9027925ba3f01c5e0048c9e37ce432 into 3415c5a2f48e4456ae57b4a4a03ea9ab(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:04,005 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:04,005 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/A, priority=12, startTime=1732762563975; duration=0sec 2024-11-28T02:56:04,005 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:04,005 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:A 2024-11-28T02:56:04,005 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:56:04,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/0bf394d3e85c4cf88c821fdc04c82da8 2024-11-28T02:56:04,011 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47156 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:56:04,011 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/C is initiating minor compaction (all files) 2024-11-28T02:56:04,012 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/C in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:04,012 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ace8d694606e4534ac8226a3a9059d14, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/9c86614e4c23484eb13a6dfb80c81fc4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/db13eff90f1446959489d3cf0b36de12, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/20b27971181c4fa2afeba0faa2818e10] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=46.1 K 2024-11-28T02:56:04,012 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/f967db3fc9ef4063b26f73440d1b5009 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/f967db3fc9ef4063b26f73440d1b5009 2024-11-28T02:56:04,012 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ace8d694606e4534ac8226a3a9059d14, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732762559756 2024-11-28T02:56:04,013 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c86614e4c23484eb13a6dfb80c81fc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732762561705 2024-11-28T02:56:04,014 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting db13eff90f1446959489d3cf0b36de12, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732762562060 2024-11-28T02:56:04,014 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20b27971181c4fa2afeba0faa2818e10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732762562728 2024-11-28T02:56:04,018 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/B of 2a9027925ba3f01c5e0048c9e37ce432 into f967db3fc9ef4063b26f73440d1b5009(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:04,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/ac0d426bb6ab498992424ec8bf6842b8 is 50, key is test_row_0/B:col10/1732762563992/Put/seqid=0 2024-11-28T02:56:04,018 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:04,018 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/B, priority=12, startTime=1732762563975; duration=0sec 2024-11-28T02:56:04,018 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:04,018 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:B 2024-11-28T02:56:04,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T02:56:04,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:04,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:04,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:04,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:04,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:04,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:04,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762624022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762624024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742333_1509 (size=12301) 2024-11-28T02:56:04,034 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#C#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:04,035 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/499be570a0eb4d9eb9c36d3f752cd8da is 50, key is test_row_0/C:col10/1732762562732/Put/seqid=0 2024-11-28T02:56:04,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/ac0d426bb6ab498992424ec8bf6842b8 2024-11-28T02:56:04,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762624026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762624026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762624031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/98c22b49194d44dbb45e5ba61249575c is 50, key is test_row_0/C:col10/1732762563992/Put/seqid=0 2024-11-28T02:56:04,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742334_1510 (size=12983) 2024-11-28T02:56:04,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742335_1511 (size=12301) 2024-11-28T02:56:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T02:56:04,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762624136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762624136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762624145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T02:56:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:04,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:04,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:04,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T02:56:04,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:04,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:04,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:04,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:04,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:04,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762624356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762624356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762624356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,452 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/499be570a0eb4d9eb9c36d3f752cd8da as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/499be570a0eb4d9eb9c36d3f752cd8da 2024-11-28T02:56:04,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/98c22b49194d44dbb45e5ba61249575c 2024-11-28T02:56:04,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/0bf394d3e85c4cf88c821fdc04c82da8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/0bf394d3e85c4cf88c821fdc04c82da8 2024-11-28T02:56:04,455 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/C of 2a9027925ba3f01c5e0048c9e37ce432 into 499be570a0eb4d9eb9c36d3f752cd8da(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:04,455 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:04,455 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/C, priority=12, startTime=1732762563975; duration=0sec 2024-11-28T02:56:04,455 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:04,455 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:C 2024-11-28T02:56:04,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/0bf394d3e85c4cf88c821fdc04c82da8, entries=200, sequenceid=345, filesize=14.4 K 2024-11-28T02:56:04,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/ac0d426bb6ab498992424ec8bf6842b8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ac0d426bb6ab498992424ec8bf6842b8 2024-11-28T02:56:04,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ac0d426bb6ab498992424ec8bf6842b8, entries=150, sequenceid=345, filesize=12.0 K 2024-11-28T02:56:04,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/98c22b49194d44dbb45e5ba61249575c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/98c22b49194d44dbb45e5ba61249575c 2024-11-28T02:56:04,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/98c22b49194d44dbb45e5ba61249575c, entries=150, sequenceid=345, filesize=12.0 K 2024-11-28T02:56:04,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 2a9027925ba3f01c5e0048c9e37ce432 in 470ms, sequenceid=345, compaction requested=false 2024-11-28T02:56:04,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:04,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T02:56:04,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:04,477 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-28T02:56:04,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:56:04,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:56:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:56:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:04,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/6404dc1d04c84f088b7aac1e6a754bbe is 50, key is test_row_0/A:col10/1732762564019/Put/seqid=0 2024-11-28T02:56:04,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742336_1512 (size=12301) 2024-11-28T02:56:04,485 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/6404dc1d04c84f088b7aac1e6a754bbe 2024-11-28T02:56:04,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/a3fae5574c134ec8910d4469f35c34f5 is 50, key is test_row_0/B:col10/1732762564019/Put/seqid=0 2024-11-28T02:56:04,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742337_1513 (size=12301) 2024-11-28T02:56:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:56:04,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:04,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762564707, max timestamp=1732762564744Num uniques 859; 2024-11-28T02:56:04,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762564707, max timestamp=1732762564744Num uniques 859; 2024-11-28T02:56:04,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:56:04,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:56:04,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762564707, max timestamp=1732762564744Num uniques 859; 2024-11-28T02:56:04,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:56:04,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762624796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762624799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762624801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,897 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/a3fae5574c134ec8910d4469f35c34f5 2024-11-28T02:56:04,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/1a1652c8f36b4a848479acece50418e8 is 50, key is test_row_0/C:col10/1732762564019/Put/seqid=0 2024-11-28T02:56:04,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742338_1514 (size=12301) 2024-11-28T02:56:04,905 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/1a1652c8f36b4a848479acece50418e8 2024-11-28T02:56:04,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/6404dc1d04c84f088b7aac1e6a754bbe as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/6404dc1d04c84f088b7aac1e6a754bbe 2024-11-28T02:56:04,911 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/6404dc1d04c84f088b7aac1e6a754bbe, entries=150, sequenceid=357, filesize=12.0 K 2024-11-28T02:56:04,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/a3fae5574c134ec8910d4469f35c34f5 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/a3fae5574c134ec8910d4469f35c34f5 2024-11-28T02:56:04,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762624906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,914 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/a3fae5574c134ec8910d4469f35c34f5, entries=150, sequenceid=357, filesize=12.0 K 2024-11-28T02:56:04,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/1a1652c8f36b4a848479acece50418e8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/1a1652c8f36b4a848479acece50418e8 2024-11-28T02:56:04,917 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/1a1652c8f36b4a848479acece50418e8, entries=150, sequenceid=357, filesize=12.0 K 2024-11-28T02:56:04,917 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 2a9027925ba3f01c5e0048c9e37ce432 in 440ms, sequenceid=357, compaction requested=true 2024-11-28T02:56:04,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:04,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:04,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-28T02:56:04,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-28T02:56:04,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-28T02:56:04,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9700 sec 2024-11-28T02:56:04,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.9760 sec 2024-11-28T02:56:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:56:04,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-28T02:56:04,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:56:04,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:04,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:56:04,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:04,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:56:04,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:04,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/44e925c7e757481188e6eb7dc159ecad is 50, key is test_row_0/A:col10/1732762564926/Put/seqid=0 2024-11-28T02:56:04,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742339_1515 (size=14741) 2024-11-28T02:56:04,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762624946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:04,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762624951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:04,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:05,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762625046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:05,057 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 2a9027925ba3f01c5e0048c9e37ce432 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:56:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:05,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:05,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762625055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:05,059 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 2a9027925ba3f01c5e0048c9e37ce432 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:56:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:05,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762625066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:05,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762625066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T02:56:05,096 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:05,097 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:56:05,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762625125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:05,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-28T02:56:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T02:56:05,149 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:56:05,150 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:56:05,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:56:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T02:56:05,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762625276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:05,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762625276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:05,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-28T02:56:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:05,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:05,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/44e925c7e757481188e6eb7dc159ecad 2024-11-28T02:56:05,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/ecdbbf6ed9fa480aba3195fac7e3582f is 50, key is test_row_0/B:col10/1732762564926/Put/seqid=0 2024-11-28T02:56:05,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742340_1516 (size=12301) 2024-11-28T02:56:05,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762625436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:05,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-28T02:56:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:05,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T02:56:05,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762625596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762625596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:05,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-28T02:56:05,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:05,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:05,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:05,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:05,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:05,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/ecdbbf6ed9fa480aba3195fac7e3582f 2024-11-28T02:56:05,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/3d32ae9ced394a37bc886f49226a326f is 50, key is test_row_0/C:col10/1732762564926/Put/seqid=0 2024-11-28T02:56:05,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742341_1517 (size=12301) 2024-11-28T02:56:05,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/3d32ae9ced394a37bc886f49226a326f 2024-11-28T02:56:05,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-28T02:56:05,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:05,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:05,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:05,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:05,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:05,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/44e925c7e757481188e6eb7dc159ecad as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/44e925c7e757481188e6eb7dc159ecad 2024-11-28T02:56:05,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/44e925c7e757481188e6eb7dc159ecad, entries=200, sequenceid=394, filesize=14.4 K 2024-11-28T02:56:05,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/ecdbbf6ed9fa480aba3195fac7e3582f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ecdbbf6ed9fa480aba3195fac7e3582f 2024-11-28T02:56:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T02:56:05,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ecdbbf6ed9fa480aba3195fac7e3582f, entries=150, sequenceid=394, filesize=12.0 K 2024-11-28T02:56:05,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/3d32ae9ced394a37bc886f49226a326f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3d32ae9ced394a37bc886f49226a326f 2024-11-28T02:56:05,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3d32ae9ced394a37bc886f49226a326f, entries=150, sequenceid=394, filesize=12.0 K 2024-11-28T02:56:05,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 2a9027925ba3f01c5e0048c9e37ce432 in 853ms, sequenceid=394, compaction requested=true 2024-11-28T02:56:05,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:05,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:56:05,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:05,781 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:56:05,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:56:05,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:05,781 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:56:05,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:56:05,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:05,781 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:56:05,781 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:56:05,781 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/B is initiating minor compaction (all files) 2024-11-28T02:56:05,781 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/A is initiating minor compaction (all files) 2024-11-28T02:56:05,782 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/B in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:05,782 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/A in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:05,782 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/f967db3fc9ef4063b26f73440d1b5009, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ac0d426bb6ab498992424ec8bf6842b8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/a3fae5574c134ec8910d4469f35c34f5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ecdbbf6ed9fa480aba3195fac7e3582f] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=48.7 K 2024-11-28T02:56:05,782 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/3415c5a2f48e4456ae57b4a4a03ea9ab, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/0bf394d3e85c4cf88c821fdc04c82da8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/6404dc1d04c84f088b7aac1e6a754bbe, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/44e925c7e757481188e6eb7dc159ecad] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=53.5 K 2024-11-28T02:56:05,782 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3415c5a2f48e4456ae57b4a4a03ea9ab, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732762562728 2024-11-28T02:56:05,782 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting f967db3fc9ef4063b26f73440d1b5009, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732762562728 2024-11-28T02:56:05,782 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bf394d3e85c4cf88c821fdc04c82da8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732762562833 2024-11-28T02:56:05,782 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting ac0d426bb6ab498992424ec8bf6842b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732762562833 2024-11-28T02:56:05,782 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6404dc1d04c84f088b7aac1e6a754bbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732762564019 2024-11-28T02:56:05,782 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting a3fae5574c134ec8910d4469f35c34f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732762564019 2024-11-28T02:56:05,782 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44e925c7e757481188e6eb7dc159ecad, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732762564783 2024-11-28T02:56:05,782 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting ecdbbf6ed9fa480aba3195fac7e3582f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732762564783 2024-11-28T02:56:05,790 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#A#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:05,791 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/adb6edd64e624ba980321feedb386dfe is 50, key is test_row_0/A:col10/1732762564926/Put/seqid=0 2024-11-28T02:56:05,791 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#B#compaction#439 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:05,792 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/747f5c842fc64e3b8472d509e581df02 is 50, key is test_row_0/B:col10/1732762564926/Put/seqid=0 2024-11-28T02:56:05,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742342_1518 (size=13119) 2024-11-28T02:56:05,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742343_1519 (size=13119) 2024-11-28T02:56:05,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-28T02:56:05,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:05,914 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T02:56:05,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:56:05,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:05,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:56:05,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:05,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:56:05,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:05,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/2fc8aaf808704e918b90c4ab95f16134 is 50, key is test_row_1/A:col10/1732762564949/Put/seqid=0 2024-11-28T02:56:05,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742344_1520 (size=9857) 2024-11-28T02:56:05,926 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/2fc8aaf808704e918b90c4ab95f16134 2024-11-28T02:56:05,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/45d92a196c304dba9fdb737b2700cdfd is 50, key is test_row_1/B:col10/1732762564949/Put/seqid=0 2024-11-28T02:56:05,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742345_1521 (size=9857) 2024-11-28T02:56:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:56:05,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:06,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762566003, max timestamp=1732762566076Num uniques 859; 2024-11-28T02:56:06,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762566003, max timestamp=1732762566076Num uniques 859; 2024-11-28T02:56:06,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732762566003, max timestamp=1732762566076Num uniques 859; 2024-11-28T02:56:06,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:56:06,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:56:06,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T02:56:06,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732762566124, max timestamp=1732762566159Num uniques 1018; 2024-11-28T02:56:06,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-28T02:56:06,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732762566124, max timestamp=1732762566159Num uniques 1018; 2024-11-28T02:56:06,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-28T02:56:06,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732762566124, max timestamp=1732762566159Num uniques 1018; 2024-11-28T02:56:06,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-28T02:56:06,203 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/adb6edd64e624ba980321feedb386dfe as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/adb6edd64e624ba980321feedb386dfe 2024-11-28T02:56:06,207 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/A of 2a9027925ba3f01c5e0048c9e37ce432 into adb6edd64e624ba980321feedb386dfe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:06,207 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:06,207 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/A, priority=12, startTime=1732762565780; duration=0sec 2024-11-28T02:56:06,207 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:06,207 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:A 2024-11-28T02:56:06,207 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:56:06,207 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/747f5c842fc64e3b8472d509e581df02 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/747f5c842fc64e3b8472d509e581df02 2024-11-28T02:56:06,208 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:56:06,208 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/C is initiating minor compaction (all files) 2024-11-28T02:56:06,208 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/C in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:06,208 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/499be570a0eb4d9eb9c36d3f752cd8da, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/98c22b49194d44dbb45e5ba61249575c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/1a1652c8f36b4a848479acece50418e8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3d32ae9ced394a37bc886f49226a326f] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=48.7 K 2024-11-28T02:56:06,208 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 499be570a0eb4d9eb9c36d3f752cd8da, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732762562728 2024-11-28T02:56:06,208 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98c22b49194d44dbb45e5ba61249575c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732762562833 2024-11-28T02:56:06,209 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a1652c8f36b4a848479acece50418e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732762564019 2024-11-28T02:56:06,209 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d32ae9ced394a37bc886f49226a326f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732762564783 2024-11-28T02:56:06,213 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/B of 2a9027925ba3f01c5e0048c9e37ce432 into 747f5c842fc64e3b8472d509e581df02(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:06,213 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:06,213 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/B, priority=12, startTime=1732762565781; duration=0sec 2024-11-28T02:56:06,213 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:06,213 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:B 2024-11-28T02:56:06,217 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#C#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:06,217 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/49bad1d671bf4e639ff9e3d332a7564e is 50, key is test_row_0/C:col10/1732762564926/Put/seqid=0 2024-11-28T02:56:06,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742346_1522 (size=13119) 2024-11-28T02:56:06,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762626226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:06,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762626227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:06,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762626230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:06,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T02:56:06,334 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/45d92a196c304dba9fdb737b2700cdfd 2024-11-28T02:56:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/5a200b30787643a98dd95ff3db413930 is 50, key is test_row_1/C:col10/1732762564949/Put/seqid=0 2024-11-28T02:56:06,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742347_1523 (size=9857) 2024-11-28T02:56:06,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762626346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:06,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762626346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:06,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762626346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:06,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762626566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:06,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762626566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:06,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762626566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:06,626 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/49bad1d671bf4e639ff9e3d332a7564e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/49bad1d671bf4e639ff9e3d332a7564e 2024-11-28T02:56:06,629 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/C of 2a9027925ba3f01c5e0048c9e37ce432 into 49bad1d671bf4e639ff9e3d332a7564e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:06,629 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:06,629 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/C, priority=12, startTime=1732762565781; duration=0sec 2024-11-28T02:56:06,629 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:06,629 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:C 2024-11-28T02:56:06,749 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/5a200b30787643a98dd95ff3db413930 2024-11-28T02:56:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/2fc8aaf808704e918b90c4ab95f16134 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/2fc8aaf808704e918b90c4ab95f16134 2024-11-28T02:56:06,756 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/2fc8aaf808704e918b90c4ab95f16134, entries=100, sequenceid=402, filesize=9.6 K 2024-11-28T02:56:06,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/45d92a196c304dba9fdb737b2700cdfd as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/45d92a196c304dba9fdb737b2700cdfd 2024-11-28T02:56:06,759 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/45d92a196c304dba9fdb737b2700cdfd, entries=100, sequenceid=402, filesize=9.6 K 2024-11-28T02:56:06,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/5a200b30787643a98dd95ff3db413930 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/5a200b30787643a98dd95ff3db413930 2024-11-28T02:56:06,762 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/5a200b30787643a98dd95ff3db413930, entries=100, sequenceid=402, filesize=9.6 K 2024-11-28T02:56:06,762 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 2a9027925ba3f01c5e0048c9e37ce432 in 848ms, sequenceid=402, compaction requested=false 2024-11-28T02:56:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:06,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:06,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-28T02:56:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-28T02:56:06,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-28T02:56:06,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-11-28T02:56:06,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.6690 sec 2024-11-28T02:56:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:56:06,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=154.31 KB heapSize=435.23 KB 2024-11-28T02:56:06,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:56:06,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T02:56:06,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:56:06,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T02:56:06,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:56:06,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T02:56:06,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/590d35ceab7b4341bee5fbcb49717311 is 50, key is test_row_0/A:col10/1732762566222/Put/seqid=0 2024-11-28T02:56:06,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742348_1524 (size=14741) 2024-11-28T02:56:06,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762626894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:06,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:06,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762626896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:06,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:06,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:06,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762626897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:06,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:06,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:07,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762627006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:07,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762627006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:07,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762627016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:07,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43124 deadline: 1732762627066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:07,075 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_0' in 2a9027925ba3f01c5e0048c9e37ce432 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:56:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:07,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43136 deadline: 1732762627086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:07,091 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 2a9027925ba3f01c5e0048c9e37ce432 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:56:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:07,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762627226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:07,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762627226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:07,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762627235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:07,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/590d35ceab7b4341bee5fbcb49717311 2024-11-28T02:56:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T02:56:07,296 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:07,297 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:56:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-28T02:56:07,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/4584687e0637402c89f1fd17cf3926a8 is 50, key is test_row_0/B:col10/1732762566222/Put/seqid=0 2024-11-28T02:56:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T02:56:07,298 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:56:07,299 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:56:07,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:56:07,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742349_1525 (size=12301) 2024-11-28T02:56:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T02:56:07,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-28T02:56:07,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:07,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:07,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:07,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:07,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:07,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762627536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:07,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762627536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:07,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:07,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:07,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762627555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:07,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-28T02:56:07,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:07,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:07,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:07,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:07,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T02:56:07,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/4584687e0637402c89f1fd17cf3926a8 2024-11-28T02:56:07,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/4225ae21d14e4333a6bd0ec27dced351 is 50, key is test_row_0/C:col10/1732762566222/Put/seqid=0 2024-11-28T02:56:07,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742350_1526 (size=12301) 2024-11-28T02:56:07,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-28T02:56:07,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:07,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:07,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:07,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:07,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:07,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:07,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-28T02:56:07,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:07,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:07,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:07,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:07,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:07,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T02:56:07,997 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T02:56:08,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:08,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43108 deadline: 1732762628045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:08,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:08,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-28T02:56:08,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:08,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. as already flushing 2024-11-28T02:56:08,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:08,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:08,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:08,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43144 deadline: 1732762628056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:08,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43172 deadline: 1732762628066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 , the old value is region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., hostname=978bd30859c3,41631,1732762419946, seqNum=2 because the exception is null or not the one we care about 2024-11-28T02:56:08,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/4225ae21d14e4333a6bd0ec27dced351 2024-11-28T02:56:08,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/590d35ceab7b4341bee5fbcb49717311 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/590d35ceab7b4341bee5fbcb49717311 2024-11-28T02:56:08,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/590d35ceab7b4341bee5fbcb49717311, entries=200, sequenceid=460, filesize=14.4 K 2024-11-28T02:56:08,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/4584687e0637402c89f1fd17cf3926a8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/4584687e0637402c89f1fd17cf3926a8 2024-11-28T02:56:08,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/4584687e0637402c89f1fd17cf3926a8, entries=150, sequenceid=460, filesize=12.0 K 2024-11-28T02:56:08,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/4225ae21d14e4333a6bd0ec27dced351 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/4225ae21d14e4333a6bd0ec27dced351 2024-11-28T02:56:08,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/4225ae21d14e4333a6bd0ec27dced351, entries=150, sequenceid=460, filesize=12.0 K 2024-11-28T02:56:08,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~435.19 KB/445632, currentSize=26.84 KB/27480 for 2a9027925ba3f01c5e0048c9e37ce432 in 1250ms, sequenceid=460, compaction requested=true 2024-11-28T02:56:08,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:08,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:56:08,131 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:08,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:08,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:56:08,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:08,131 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:08,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a9027925ba3f01c5e0048c9e37ce432:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:56:08,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:08,132 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:08,132 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/B is initiating minor compaction (all files) 2024-11-28T02:56:08,132 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/B in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:08,132 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/747f5c842fc64e3b8472d509e581df02, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/45d92a196c304dba9fdb737b2700cdfd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/4584687e0637402c89f1fd17cf3926a8] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=34.5 K 2024-11-28T02:56:08,132 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37717 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:08,132 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/A is initiating minor compaction (all files) 2024-11-28T02:56:08,132 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/A in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:08,132 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/adb6edd64e624ba980321feedb386dfe, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/2fc8aaf808704e918b90c4ab95f16134, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/590d35ceab7b4341bee5fbcb49717311] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=36.8 K 2024-11-28T02:56:08,133 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 747f5c842fc64e3b8472d509e581df02, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732762564783 2024-11-28T02:56:08,133 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting adb6edd64e624ba980321feedb386dfe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732762564783 2024-11-28T02:56:08,133 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 45d92a196c304dba9fdb737b2700cdfd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732762564943 2024-11-28T02:56:08,133 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fc8aaf808704e918b90c4ab95f16134, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732762564943 2024-11-28T02:56:08,133 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4584687e0637402c89f1fd17cf3926a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732762566222 2024-11-28T02:56:08,133 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 590d35ceab7b4341bee5fbcb49717311, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732762566222 2024-11-28T02:56:08,139 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#A#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:08,140 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/e4d47fac271a4475bb5f56d0d133abcd is 50, key is test_row_0/A:col10/1732762566222/Put/seqid=0 2024-11-28T02:56:08,143 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#B#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:08,144 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/38e2a2b4a66c4df5a7606749346a2bf4 is 50, key is test_row_0/B:col10/1732762566222/Put/seqid=0 2024-11-28T02:56:08,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742351_1527 (size=13221) 2024-11-28T02:56:08,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742352_1528 (size=13221) 2024-11-28T02:56:08,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-28T02:56:08,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:08,216 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-28T02:56:08,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:56:08,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:08,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:56:08,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:08,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:56:08,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:08,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/fc8b4c7a61a4479692dab5e5e14cb349 is 50, key is test_row_1/A:col10/1732762566881/Put/seqid=0 2024-11-28T02:56:08,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742353_1529 (size=9857) 2024-11-28T02:56:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T02:56:08,553 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/e4d47fac271a4475bb5f56d0d133abcd as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/e4d47fac271a4475bb5f56d0d133abcd 2024-11-28T02:56:08,557 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/A of 2a9027925ba3f01c5e0048c9e37ce432 into e4d47fac271a4475bb5f56d0d133abcd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:08,557 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:08,557 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/A, priority=13, startTime=1732762568130; duration=0sec 2024-11-28T02:56:08,557 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:08,557 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:A 2024-11-28T02:56:08,558 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:08,558 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:08,558 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 2a9027925ba3f01c5e0048c9e37ce432/C is initiating minor compaction (all files) 2024-11-28T02:56:08,558 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a9027925ba3f01c5e0048c9e37ce432/C in TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:08,558 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/49bad1d671bf4e639ff9e3d332a7564e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/5a200b30787643a98dd95ff3db413930, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/4225ae21d14e4333a6bd0ec27dced351] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp, totalSize=34.5 K 2024-11-28T02:56:08,559 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49bad1d671bf4e639ff9e3d332a7564e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732762564783 2024-11-28T02:56:08,559 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a200b30787643a98dd95ff3db413930, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732762564943 2024-11-28T02:56:08,559 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4225ae21d14e4333a6bd0ec27dced351, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732762566222 2024-11-28T02:56:08,564 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a9027925ba3f01c5e0048c9e37ce432#C#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:08,565 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/38e2a2b4a66c4df5a7606749346a2bf4 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/38e2a2b4a66c4df5a7606749346a2bf4 2024-11-28T02:56:08,565 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/cf247d6b4d6b4b5a917e699401592c9f is 50, key is test_row_0/C:col10/1732762566222/Put/seqid=0 2024-11-28T02:56:08,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742354_1530 (size=13221) 2024-11-28T02:56:08,569 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/B of 2a9027925ba3f01c5e0048c9e37ce432 into 38e2a2b4a66c4df5a7606749346a2bf4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:08,569 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:08,569 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/B, priority=13, startTime=1732762568131; duration=0sec 2024-11-28T02:56:08,569 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:08,569 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:B 2024-11-28T02:56:08,624 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/fc8b4c7a61a4479692dab5e5e14cb349 2024-11-28T02:56:08,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/fbba724157f14635adfd0cf85a0542fe is 50, key is test_row_1/B:col10/1732762566881/Put/seqid=0 2024-11-28T02:56:08,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742355_1531 (size=9857) 2024-11-28T02:56:08,637 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/fbba724157f14635adfd0cf85a0542fe 2024-11-28T02:56:08,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/997dbbd75f7e47ba8dc51cedfb681d4e is 50, key is test_row_1/C:col10/1732762566881/Put/seqid=0 2024-11-28T02:56:08,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742356_1532 (size=9857) 2024-11-28T02:56:08,973 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/cf247d6b4d6b4b5a917e699401592c9f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/cf247d6b4d6b4b5a917e699401592c9f 2024-11-28T02:56:08,978 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a9027925ba3f01c5e0048c9e37ce432/C of 2a9027925ba3f01c5e0048c9e37ce432 into cf247d6b4d6b4b5a917e699401592c9f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:08,978 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:08,978 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., storeName=2a9027925ba3f01c5e0048c9e37ce432/C, priority=13, startTime=1732762568131; duration=0sec 2024-11-28T02:56:08,978 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:08,978 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a9027925ba3f01c5e0048c9e37ce432:C 2024-11-28T02:56:09,024 INFO [Thread-2018 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2018. 2024-11-28T02:56:09,024 DEBUG [Thread-2018 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:09,024 DEBUG [Thread-2018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,025 INFO [Thread-2016 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2016. 2024-11-28T02:56:09,025 DEBUG [Thread-2016 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:09,025 DEBUG [Thread-2016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,025 DEBUG [Thread-2016 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,025 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:09,026 DEBUG [Thread-2018 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:09,026 INFO [Thread-2020 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2020. 2024-11-28T02:56:09,026 DEBUG [Thread-2020 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:09,026 DEBUG [Thread-2020 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:09,026 DEBUG [Thread-2020 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,028 INFO [Thread-2024 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2024. 2024-11-28T02:56:09,028 DEBUG [Thread-2024 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:09,028 DEBUG [Thread-2024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,028 DEBUG [Thread-2024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:09,029 INFO [Thread-2022 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2022. 2024-11-28T02:56:09,029 DEBUG [Thread-2022 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:09,029 DEBUG [Thread-2022 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,029 DEBUG [Thread-2022 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:09,059 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/997dbbd75f7e47ba8dc51cedfb681d4e 2024-11-28T02:56:09,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/fc8b4c7a61a4479692dab5e5e14cb349 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/fc8b4c7a61a4479692dab5e5e14cb349 2024-11-28T02:56:09,067 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/fc8b4c7a61a4479692dab5e5e14cb349, entries=100, sequenceid=467, filesize=9.6 K 2024-11-28T02:56:09,068 INFO [Thread-2007 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2007. 2024-11-28T02:56:09,068 DEBUG [Thread-2007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:09,068 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/fbba724157f14635adfd0cf85a0542fe as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/fbba724157f14635adfd0cf85a0542fe 2024-11-28T02:56:09,068 DEBUG [Thread-2007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:09,071 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/fbba724157f14635adfd0cf85a0542fe, entries=100, sequenceid=467, filesize=9.6 K 2024-11-28T02:56:09,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/997dbbd75f7e47ba8dc51cedfb681d4e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/997dbbd75f7e47ba8dc51cedfb681d4e 2024-11-28T02:56:09,075 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/997dbbd75f7e47ba8dc51cedfb681d4e, entries=100, sequenceid=467, filesize=9.6 K 2024-11-28T02:56:09,075 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=6.71 KB/6870 for 2a9027925ba3f01c5e0048c9e37ce432 in 859ms, sequenceid=467, compaction requested=false 2024-11-28T02:56:09,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 2a9027925ba3f01c5e0048c9e37ce432: 2024-11-28T02:56:09,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:09,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-28T02:56:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-28T02:56:09,078 INFO [Thread-2005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2005. 2024-11-28T02:56:09,078 DEBUG [Thread-2005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:09,078 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,078 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:09,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-28T02:56:09,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7770 sec 2024-11-28T02:56:09,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.7810 sec 2024-11-28T02:56:09,087 INFO [Thread-2013 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2013. 2024-11-28T02:56:09,087 DEBUG [Thread-2013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:09,087 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,087 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:09,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T02:56:09,446 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:11,102 INFO [Thread-2011 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2011. 2024-11-28T02:56:11,102 DEBUG [Thread-2011 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:11,102 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:11,102 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:11,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:11,138 INFO [Thread-2009 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2009. 2024-11-28T02:56:11,138 DEBUG [Thread-2009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:11,139 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:11,139 DEBUG [Thread-2009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:11,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:11,139 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T02:56:11,139 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-28T02:56:11,139 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-28T02:56:11,139 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-28T02:56:11,139 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-28T02:56:11,139 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-28T02:56:11,139 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T02:56:11,139 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T02:56:11,139 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1283 2024-11-28T02:56:11,139 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3849 rows 2024-11-28T02:56:11,139 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1291 2024-11-28T02:56:11,139 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3869 rows 2024-11-28T02:56:11,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1292 2024-11-28T02:56:11,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3876 rows 2024-11-28T02:56:11,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1286 2024-11-28T02:56:11,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3853 rows 2024-11-28T02:56:11,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1289 2024-11-28T02:56:11,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3867 rows 2024-11-28T02:56:11,140 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:56:11,140 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:56:11,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:11,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:11,140 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:11,142 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T02:56:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-28T02:56:11,147 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762571147"}]},"ts":"1732762571147"} 2024-11-28T02:56:11,149 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T02:56:11,149 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T02:56:11,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T02:56:11,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a9027925ba3f01c5e0048c9e37ce432, UNASSIGN}] 2024-11-28T02:56:11,152 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a9027925ba3f01c5e0048c9e37ce432, UNASSIGN 2024-11-28T02:56:11,153 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=2a9027925ba3f01c5e0048c9e37ce432, regionState=CLOSING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:56:11,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a9027925ba3f01c5e0048c9e37ce432, UNASSIGN because future has completed 2024-11-28T02:56:11,155 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T02:56:11,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:56:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-28T02:56:11,308 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:56:11,308 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T02:56:11,308 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 2a9027925ba3f01c5e0048c9e37ce432, disabling compactions & flushes 2024-11-28T02:56:11,308 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:11,308 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:11,308 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. after waiting 0 ms 2024-11-28T02:56:11,308 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:11,308 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(2902): Flushing 2a9027925ba3f01c5e0048c9e37ce432 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T02:56:11,309 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=A 2024-11-28T02:56:11,309 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:11,309 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=B 2024-11-28T02:56:11,309 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:11,309 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2a9027925ba3f01c5e0048c9e37ce432, store=C 2024-11-28T02:56:11,309 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:11,315 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/a0318829027046e48c17f5bbe418db24 is 50, key is test_row_0/A:col10/1732762571098/Put/seqid=0 2024-11-28T02:56:11,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742357_1533 (size=12301) 2024-11-28T02:56:11,320 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/a0318829027046e48c17f5bbe418db24 2024-11-28T02:56:11,327 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/133c838a34004960a20e7143cff3b7b2 is 50, key is test_row_0/B:col10/1732762571098/Put/seqid=0 2024-11-28T02:56:11,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742358_1534 (size=12301) 2024-11-28T02:56:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-28T02:56:11,732 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/133c838a34004960a20e7143cff3b7b2 2024-11-28T02:56:11,742 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/40035c72cdd34176a2914323f5b90a6a is 50, key is test_row_0/C:col10/1732762571098/Put/seqid=0 2024-11-28T02:56:11,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742359_1535 (size=12301) 2024-11-28T02:56:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-28T02:56:12,148 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/40035c72cdd34176a2914323f5b90a6a 2024-11-28T02:56:12,153 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/A/a0318829027046e48c17f5bbe418db24 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/a0318829027046e48c17f5bbe418db24 2024-11-28T02:56:12,159 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/a0318829027046e48c17f5bbe418db24, entries=150, sequenceid=478, filesize=12.0 K 2024-11-28T02:56:12,160 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/B/133c838a34004960a20e7143cff3b7b2 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/133c838a34004960a20e7143cff3b7b2 2024-11-28T02:56:12,165 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/133c838a34004960a20e7143cff3b7b2, entries=150, sequenceid=478, filesize=12.0 K 2024-11-28T02:56:12,165 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/.tmp/C/40035c72cdd34176a2914323f5b90a6a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/40035c72cdd34176a2914323f5b90a6a 2024-11-28T02:56:12,169 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/40035c72cdd34176a2914323f5b90a6a, entries=150, sequenceid=478, filesize=12.0 K 2024-11-28T02:56:12,170 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2a9027925ba3f01c5e0048c9e37ce432 in 862ms, sequenceid=478, compaction requested=true 2024-11-28T02:56:12,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/95ced3ebc8b44e2daa21879028f47f44, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bae16dccdc46408e9bc44b79656b1ef1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/f2a4840e145f4748a392cede008a51de, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/c75291c2a7ae4d55895f45a61588b473, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/21739f595d2347bb854ac0134b62400e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/de33733f48554272a0494744e47c824d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/8257ffcd34464e9cbf937033768ac761, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/184e8860d53a470eb05c35342455e5bd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bb6e9f692cf34fa08043ddcad75d9802, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/9c5f22afaaab484aa69e2e20ef7f719f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/1cc276baf43a4de1bf047fa71b39d337, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/5c5ffeca43524355a03711d461bd0356, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/92cf7242ef9e4089ac389603294bbd40, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/ff23f74dba3249468227bf027283ed25, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/b9a6c76d0d91490195519adc824ce91e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/4e856fb765eb45faaa45d9119637b0e8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/c66f91b89d164a3f8606fab3d560f1df, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/8fe6433478bd4165a6813f96b6875245, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/761cc95693754e7ca5855fce38ab1c65, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bb99d09d58f74989b8f55f7c0c29e398, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/3415c5a2f48e4456ae57b4a4a03ea9ab, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/0bf394d3e85c4cf88c821fdc04c82da8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/6404dc1d04c84f088b7aac1e6a754bbe, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/44e925c7e757481188e6eb7dc159ecad, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/adb6edd64e624ba980321feedb386dfe, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/2fc8aaf808704e918b90c4ab95f16134, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/590d35ceab7b4341bee5fbcb49717311] to archive 2024-11-28T02:56:12,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:56:12,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/95ced3ebc8b44e2daa21879028f47f44 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/95ced3ebc8b44e2daa21879028f47f44 2024-11-28T02:56:12,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bae16dccdc46408e9bc44b79656b1ef1 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bae16dccdc46408e9bc44b79656b1ef1 2024-11-28T02:56:12,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/f2a4840e145f4748a392cede008a51de to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/f2a4840e145f4748a392cede008a51de 2024-11-28T02:56:12,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/c75291c2a7ae4d55895f45a61588b473 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/c75291c2a7ae4d55895f45a61588b473 2024-11-28T02:56:12,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/21739f595d2347bb854ac0134b62400e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/21739f595d2347bb854ac0134b62400e 2024-11-28T02:56:12,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/de33733f48554272a0494744e47c824d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/de33733f48554272a0494744e47c824d 2024-11-28T02:56:12,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/8257ffcd34464e9cbf937033768ac761 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/8257ffcd34464e9cbf937033768ac761 2024-11-28T02:56:12,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/184e8860d53a470eb05c35342455e5bd to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/184e8860d53a470eb05c35342455e5bd 2024-11-28T02:56:12,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bb6e9f692cf34fa08043ddcad75d9802 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bb6e9f692cf34fa08043ddcad75d9802 2024-11-28T02:56:12,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/9c5f22afaaab484aa69e2e20ef7f719f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/9c5f22afaaab484aa69e2e20ef7f719f 2024-11-28T02:56:12,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/1cc276baf43a4de1bf047fa71b39d337 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/1cc276baf43a4de1bf047fa71b39d337 2024-11-28T02:56:12,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/5c5ffeca43524355a03711d461bd0356 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/5c5ffeca43524355a03711d461bd0356 2024-11-28T02:56:12,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/92cf7242ef9e4089ac389603294bbd40 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/92cf7242ef9e4089ac389603294bbd40 2024-11-28T02:56:12,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/ff23f74dba3249468227bf027283ed25 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/ff23f74dba3249468227bf027283ed25 2024-11-28T02:56:12,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/b9a6c76d0d91490195519adc824ce91e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/b9a6c76d0d91490195519adc824ce91e 2024-11-28T02:56:12,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/4e856fb765eb45faaa45d9119637b0e8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/4e856fb765eb45faaa45d9119637b0e8 2024-11-28T02:56:12,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/c66f91b89d164a3f8606fab3d560f1df to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/c66f91b89d164a3f8606fab3d560f1df 2024-11-28T02:56:12,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/8fe6433478bd4165a6813f96b6875245 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/8fe6433478bd4165a6813f96b6875245 2024-11-28T02:56:12,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/761cc95693754e7ca5855fce38ab1c65 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/761cc95693754e7ca5855fce38ab1c65 2024-11-28T02:56:12,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bb99d09d58f74989b8f55f7c0c29e398 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/bb99d09d58f74989b8f55f7c0c29e398 2024-11-28T02:56:12,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/3415c5a2f48e4456ae57b4a4a03ea9ab to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/3415c5a2f48e4456ae57b4a4a03ea9ab 2024-11-28T02:56:12,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/0bf394d3e85c4cf88c821fdc04c82da8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/0bf394d3e85c4cf88c821fdc04c82da8 2024-11-28T02:56:12,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/6404dc1d04c84f088b7aac1e6a754bbe to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/6404dc1d04c84f088b7aac1e6a754bbe 2024-11-28T02:56:12,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/44e925c7e757481188e6eb7dc159ecad to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/44e925c7e757481188e6eb7dc159ecad 2024-11-28T02:56:12,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/adb6edd64e624ba980321feedb386dfe to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/adb6edd64e624ba980321feedb386dfe 2024-11-28T02:56:12,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/2fc8aaf808704e918b90c4ab95f16134 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/2fc8aaf808704e918b90c4ab95f16134 2024-11-28T02:56:12,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/590d35ceab7b4341bee5fbcb49717311 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/590d35ceab7b4341bee5fbcb49717311 2024-11-28T02:56:12,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/9724609cf03749ef8e28b78bd1d181ef, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/e33fba4a0f034c07a427b5e2200daa21, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/84f2b16dfb764206950c9a21ba1f28ef, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/78276b55e09c496fb2172dcdfdb58b48, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/5b85e13e972d48a78928247d9b957a2f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c4c4cde0c3a74a9dbece855550499050, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3f605dc0d7a640ca88d2826cd627502a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/8b2edaf186004893b617dae49131ee73, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c6973917ac6b4ec4bbcf7511b3c24af4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/cbef5792c243497598eed9c769a7c0b8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c6c285a499214e2c8f44bd52664ec07c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3411c7f1c6e745ea9ce324b5ca9a01e0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/922bf86c9ae246f0bec6393f5ba39272, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c5c1c1ea69594b7581e0a453f4ead424, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/21d7893aeaca496f8c639eaa21fa3387, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ccd3dc4b0dd84ca6b66eb01a6b4b008c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/437f2e01f91a46f0863c6d230466214a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/e2baf4916da6478bbae06984e0ddf6e5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/db97bcea32334ed8b19c969fdc8ad0d2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/f967db3fc9ef4063b26f73440d1b5009, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3edf6940d637484dbc16f544f40ba672, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ac0d426bb6ab498992424ec8bf6842b8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/a3fae5574c134ec8910d4469f35c34f5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/747f5c842fc64e3b8472d509e581df02, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ecdbbf6ed9fa480aba3195fac7e3582f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/45d92a196c304dba9fdb737b2700cdfd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/4584687e0637402c89f1fd17cf3926a8] to archive 2024-11-28T02:56:12,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:56:12,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/9724609cf03749ef8e28b78bd1d181ef to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/9724609cf03749ef8e28b78bd1d181ef 2024-11-28T02:56:12,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/e33fba4a0f034c07a427b5e2200daa21 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/e33fba4a0f034c07a427b5e2200daa21 2024-11-28T02:56:12,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/84f2b16dfb764206950c9a21ba1f28ef to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/84f2b16dfb764206950c9a21ba1f28ef 2024-11-28T02:56:12,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/78276b55e09c496fb2172dcdfdb58b48 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/78276b55e09c496fb2172dcdfdb58b48 2024-11-28T02:56:12,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/5b85e13e972d48a78928247d9b957a2f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/5b85e13e972d48a78928247d9b957a2f 2024-11-28T02:56:12,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c4c4cde0c3a74a9dbece855550499050 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c4c4cde0c3a74a9dbece855550499050 2024-11-28T02:56:12,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3f605dc0d7a640ca88d2826cd627502a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3f605dc0d7a640ca88d2826cd627502a 2024-11-28T02:56:12,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/8b2edaf186004893b617dae49131ee73 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/8b2edaf186004893b617dae49131ee73 2024-11-28T02:56:12,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c6973917ac6b4ec4bbcf7511b3c24af4 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c6973917ac6b4ec4bbcf7511b3c24af4 2024-11-28T02:56:12,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/cbef5792c243497598eed9c769a7c0b8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/cbef5792c243497598eed9c769a7c0b8 2024-11-28T02:56:12,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c6c285a499214e2c8f44bd52664ec07c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c6c285a499214e2c8f44bd52664ec07c 2024-11-28T02:56:12,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3411c7f1c6e745ea9ce324b5ca9a01e0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3411c7f1c6e745ea9ce324b5ca9a01e0 2024-11-28T02:56:12,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/922bf86c9ae246f0bec6393f5ba39272 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/922bf86c9ae246f0bec6393f5ba39272 2024-11-28T02:56:12,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c5c1c1ea69594b7581e0a453f4ead424 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/c5c1c1ea69594b7581e0a453f4ead424 2024-11-28T02:56:12,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/21d7893aeaca496f8c639eaa21fa3387 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/21d7893aeaca496f8c639eaa21fa3387 2024-11-28T02:56:12,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ccd3dc4b0dd84ca6b66eb01a6b4b008c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ccd3dc4b0dd84ca6b66eb01a6b4b008c 2024-11-28T02:56:12,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/437f2e01f91a46f0863c6d230466214a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/437f2e01f91a46f0863c6d230466214a 2024-11-28T02:56:12,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/e2baf4916da6478bbae06984e0ddf6e5 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/e2baf4916da6478bbae06984e0ddf6e5 2024-11-28T02:56:12,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/db97bcea32334ed8b19c969fdc8ad0d2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/db97bcea32334ed8b19c969fdc8ad0d2 2024-11-28T02:56:12,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/f967db3fc9ef4063b26f73440d1b5009 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/f967db3fc9ef4063b26f73440d1b5009 2024-11-28T02:56:12,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3edf6940d637484dbc16f544f40ba672 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/3edf6940d637484dbc16f544f40ba672 2024-11-28T02:56:12,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ac0d426bb6ab498992424ec8bf6842b8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ac0d426bb6ab498992424ec8bf6842b8 2024-11-28T02:56:12,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/a3fae5574c134ec8910d4469f35c34f5 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/a3fae5574c134ec8910d4469f35c34f5 2024-11-28T02:56:12,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/747f5c842fc64e3b8472d509e581df02 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/747f5c842fc64e3b8472d509e581df02 2024-11-28T02:56:12,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ecdbbf6ed9fa480aba3195fac7e3582f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/ecdbbf6ed9fa480aba3195fac7e3582f 2024-11-28T02:56:12,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/45d92a196c304dba9fdb737b2700cdfd to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/45d92a196c304dba9fdb737b2700cdfd 2024-11-28T02:56:12,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/4584687e0637402c89f1fd17cf3926a8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/4584687e0637402c89f1fd17cf3926a8 2024-11-28T02:56:12,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/d06e9c0f3e524d309d5e16714b5932da, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/c3c665a583394eb6b9ade358b67e98c6, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/5e1b789af2f840db95504513ba4b9443, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ea00c7936ba541e0a7d696d007e7eb7f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3d24ac405103405d814cfbddae97167e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/31f22aa7f7c94085b4618789f2047e6f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/f95c84568b10465ea26d18cd0037f243, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ad7eac5dbde744ac9ede5257bd9c2eac, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/25f628f5f2214968a3372f4f032e579a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/6ca323bc464044959e317bf9002accde, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/2da244a0a251445cb23a203d81443c83, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/03ba03fc8eaa4b629e047df0a30d1a65, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/f37983b644d84893b3da39e0bb7912de, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/2d47307640624ac3a1f7cda78b9fc886, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/dd8aefbb7154409dbef0f646a5799abc, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ace8d694606e4534ac8226a3a9059d14, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3b2dd2033b244c4f8816021de9dcebab, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/9c86614e4c23484eb13a6dfb80c81fc4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/db13eff90f1446959489d3cf0b36de12, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/499be570a0eb4d9eb9c36d3f752cd8da, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/20b27971181c4fa2afeba0faa2818e10, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/98c22b49194d44dbb45e5ba61249575c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/1a1652c8f36b4a848479acece50418e8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/49bad1d671bf4e639ff9e3d332a7564e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3d32ae9ced394a37bc886f49226a326f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/5a200b30787643a98dd95ff3db413930, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/4225ae21d14e4333a6bd0ec27dced351] to archive 2024-11-28T02:56:12,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:56:12,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/d06e9c0f3e524d309d5e16714b5932da to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/d06e9c0f3e524d309d5e16714b5932da 2024-11-28T02:56:12,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/c3c665a583394eb6b9ade358b67e98c6 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/c3c665a583394eb6b9ade358b67e98c6 2024-11-28T02:56:12,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/5e1b789af2f840db95504513ba4b9443 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/5e1b789af2f840db95504513ba4b9443 2024-11-28T02:56:12,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ea00c7936ba541e0a7d696d007e7eb7f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ea00c7936ba541e0a7d696d007e7eb7f 2024-11-28T02:56:12,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3d24ac405103405d814cfbddae97167e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3d24ac405103405d814cfbddae97167e 2024-11-28T02:56:12,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/31f22aa7f7c94085b4618789f2047e6f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/31f22aa7f7c94085b4618789f2047e6f 2024-11-28T02:56:12,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/f95c84568b10465ea26d18cd0037f243 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/f95c84568b10465ea26d18cd0037f243 2024-11-28T02:56:12,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ad7eac5dbde744ac9ede5257bd9c2eac to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ad7eac5dbde744ac9ede5257bd9c2eac 2024-11-28T02:56:12,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/25f628f5f2214968a3372f4f032e579a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/25f628f5f2214968a3372f4f032e579a 2024-11-28T02:56:12,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/6ca323bc464044959e317bf9002accde to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/6ca323bc464044959e317bf9002accde 2024-11-28T02:56:12,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/2da244a0a251445cb23a203d81443c83 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/2da244a0a251445cb23a203d81443c83 2024-11-28T02:56:12,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/03ba03fc8eaa4b629e047df0a30d1a65 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/03ba03fc8eaa4b629e047df0a30d1a65 2024-11-28T02:56:12,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/f37983b644d84893b3da39e0bb7912de to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/f37983b644d84893b3da39e0bb7912de 2024-11-28T02:56:12,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/2d47307640624ac3a1f7cda78b9fc886 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/2d47307640624ac3a1f7cda78b9fc886 2024-11-28T02:56:12,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/dd8aefbb7154409dbef0f646a5799abc to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/dd8aefbb7154409dbef0f646a5799abc 2024-11-28T02:56:12,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ace8d694606e4534ac8226a3a9059d14 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/ace8d694606e4534ac8226a3a9059d14 2024-11-28T02:56:12,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3b2dd2033b244c4f8816021de9dcebab to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3b2dd2033b244c4f8816021de9dcebab 2024-11-28T02:56:12,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/9c86614e4c23484eb13a6dfb80c81fc4 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/9c86614e4c23484eb13a6dfb80c81fc4 2024-11-28T02:56:12,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/db13eff90f1446959489d3cf0b36de12 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/db13eff90f1446959489d3cf0b36de12 2024-11-28T02:56:12,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/499be570a0eb4d9eb9c36d3f752cd8da to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/499be570a0eb4d9eb9c36d3f752cd8da 2024-11-28T02:56:12,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/20b27971181c4fa2afeba0faa2818e10 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/20b27971181c4fa2afeba0faa2818e10 2024-11-28T02:56:12,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/98c22b49194d44dbb45e5ba61249575c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/98c22b49194d44dbb45e5ba61249575c 2024-11-28T02:56:12,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/1a1652c8f36b4a848479acece50418e8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/1a1652c8f36b4a848479acece50418e8 2024-11-28T02:56:12,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/49bad1d671bf4e639ff9e3d332a7564e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/49bad1d671bf4e639ff9e3d332a7564e 2024-11-28T02:56:12,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3d32ae9ced394a37bc886f49226a326f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/3d32ae9ced394a37bc886f49226a326f 2024-11-28T02:56:12,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/5a200b30787643a98dd95ff3db413930 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/5a200b30787643a98dd95ff3db413930 2024-11-28T02:56:12,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/4225ae21d14e4333a6bd0ec27dced351 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/4225ae21d14e4333a6bd0ec27dced351 2024-11-28T02:56:12,257 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/recovered.edits/481.seqid, newMaxSeqId=481, maxSeqId=1 2024-11-28T02:56:12,258 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. 2024-11-28T02:56:12,258 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 2a9027925ba3f01c5e0048c9e37ce432: Waiting for close lock at 1732762571308Running coprocessor pre-close hooks at 1732762571308Disabling compacts and flushes for region at 1732762571308Disabling writes for close at 1732762571308Obtaining lock to block concurrent updates at 1732762571308Preparing flush snapshotting stores in 2a9027925ba3f01c5e0048c9e37ce432 at 1732762571308Finished memstore snapshotting TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732762571309 (+1 ms)Flushing stores of TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432. at 1732762571310 (+1 ms)Flushing 2a9027925ba3f01c5e0048c9e37ce432/A: creating writer at 1732762571310Flushing 2a9027925ba3f01c5e0048c9e37ce432/A: appending metadata at 1732762571314 (+4 ms)Flushing 2a9027925ba3f01c5e0048c9e37ce432/A: closing flushed file at 1732762571314Flushing 2a9027925ba3f01c5e0048c9e37ce432/B: creating writer at 1732762571324 (+10 ms)Flushing 2a9027925ba3f01c5e0048c9e37ce432/B: appending metadata at 1732762571327 (+3 ms)Flushing 2a9027925ba3f01c5e0048c9e37ce432/B: closing flushed file at 1732762571327Flushing 2a9027925ba3f01c5e0048c9e37ce432/C: creating writer at 1732762571738 (+411 ms)Flushing 2a9027925ba3f01c5e0048c9e37ce432/C: appending metadata at 1732762571741 (+3 ms)Flushing 2a9027925ba3f01c5e0048c9e37ce432/C: closing flushed file at 1732762571742 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58d42d06: reopening flushed file at 1732762572152 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b12ffa7: reopening flushed file at 1732762572159 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@371e2dee: reopening flushed file at 1732762572165 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2a9027925ba3f01c5e0048c9e37ce432 in 862ms, sequenceid=478, compaction requested=true at 1732762572170 (+5 ms)Writing region close event to WAL at 1732762572254 (+84 ms)Running coprocessor post-close hooks at 1732762572258 (+4 ms)Closed at 1732762572258 2024-11-28T02:56:12,260 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:56:12,260 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=2a9027925ba3f01c5e0048c9e37ce432, regionState=CLOSED 2024-11-28T02:56:12,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:56:12,264 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-28T02:56:12,264 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 2a9027925ba3f01c5e0048c9e37ce432, server=978bd30859c3,41631,1732762419946 in 1.1070 sec 2024-11-28T02:56:12,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-28T02:56:12,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a9027925ba3f01c5e0048c9e37ce432, UNASSIGN in 1.1130 sec 2024-11-28T02:56:12,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-28T02:56:12,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1160 sec 2024-11-28T02:56:12,268 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762572268"}]},"ts":"1732762572268"} 2024-11-28T02:56:12,270 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T02:56:12,270 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T02:56:12,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1280 sec 2024-11-28T02:56:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-28T02:56:12,286 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:12,287 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T02:56:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:12,290 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:12,291 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=147, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-28T02:56:12,294 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:56:12,298 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/recovered.edits] 2024-11-28T02:56:12,303 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/a0318829027046e48c17f5bbe418db24 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/a0318829027046e48c17f5bbe418db24 2024-11-28T02:56:12,305 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/e4d47fac271a4475bb5f56d0d133abcd to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/e4d47fac271a4475bb5f56d0d133abcd 2024-11-28T02:56:12,308 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/fc8b4c7a61a4479692dab5e5e14cb349 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/A/fc8b4c7a61a4479692dab5e5e14cb349 2024-11-28T02:56:12,311 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/133c838a34004960a20e7143cff3b7b2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/133c838a34004960a20e7143cff3b7b2 2024-11-28T02:56:12,313 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/38e2a2b4a66c4df5a7606749346a2bf4 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/38e2a2b4a66c4df5a7606749346a2bf4 2024-11-28T02:56:12,314 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/fbba724157f14635adfd0cf85a0542fe to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/B/fbba724157f14635adfd0cf85a0542fe 2024-11-28T02:56:12,316 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/40035c72cdd34176a2914323f5b90a6a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/40035c72cdd34176a2914323f5b90a6a 2024-11-28T02:56:12,316 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/997dbbd75f7e47ba8dc51cedfb681d4e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/997dbbd75f7e47ba8dc51cedfb681d4e 2024-11-28T02:56:12,317 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/cf247d6b4d6b4b5a917e699401592c9f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/C/cf247d6b4d6b4b5a917e699401592c9f 2024-11-28T02:56:12,318 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/recovered.edits/481.seqid to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432/recovered.edits/481.seqid 2024-11-28T02:56:12,319 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/2a9027925ba3f01c5e0048c9e37ce432 2024-11-28T02:56:12,319 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T02:56:12,321 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=147, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:12,322 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T02:56:12,324 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T02:56:12,325 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=147, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:12,325 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T02:56:12,325 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732762572325"}]},"ts":"9223372036854775807"} 2024-11-28T02:56:12,326 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T02:56:12,326 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2a9027925ba3f01c5e0048c9e37ce432, NAME => 'TestAcidGuarantees,,1732762546805.2a9027925ba3f01c5e0048c9e37ce432.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T02:56:12,327 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T02:56:12,327 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732762572327"}]},"ts":"9223372036854775807"} 2024-11-28T02:56:12,328 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T02:56:12,329 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=147, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:12,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-28T02:56:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-28T02:56:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T02:56:12,397 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:12,415 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251 (was 252), OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=481 (was 540), ProcessCount=11 (was 11), AvailableMemoryMB=7231 (was 7435) 2024-11-28T02:56:12,423 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=481, ProcessCount=11, AvailableMemoryMB=7231 2024-11-28T02:56:12,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T02:56:12,424 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:56:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:12,426 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T02:56:12,426 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:12,426 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 148 2024-11-28T02:56:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-28T02:56:12,427 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T02:56:12,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742360_1536 (size=1061) 2024-11-28T02:56:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-28T02:56:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-28T02:56:12,833 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381 2024-11-28T02:56:12,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742361_1537 (size=53) 2024-11-28T02:56:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-28T02:56:13,239 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:56:13,239 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3bd4185211d8fb1e40f0aca1aa4a4576, disabling compactions & flushes 2024-11-28T02:56:13,239 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:13,239 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:13,239 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. after waiting 0 ms 2024-11-28T02:56:13,239 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:13,239 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:13,239 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3bd4185211d8fb1e40f0aca1aa4a4576: Waiting for close lock at 1732762573239Disabling compacts and flushes for region at 1732762573239Disabling writes for close at 1732762573239Writing region close event to WAL at 1732762573239Closed at 1732762573239 2024-11-28T02:56:13,241 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T02:56:13,242 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732762573242"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732762573242"}]},"ts":"1732762573242"} 2024-11-28T02:56:13,245 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T02:56:13,246 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T02:56:13,246 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762573246"}]},"ts":"1732762573246"} 2024-11-28T02:56:13,249 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T02:56:13,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, ASSIGN}] 2024-11-28T02:56:13,251 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, ASSIGN 2024-11-28T02:56:13,252 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, ASSIGN; state=OFFLINE, location=978bd30859c3,41631,1732762419946; forceNewPlan=false, retain=false 2024-11-28T02:56:13,403 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=3bd4185211d8fb1e40f0aca1aa4a4576, regionState=OPENING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:56:13,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, ASSIGN because future has completed 2024-11-28T02:56:13,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:56:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-28T02:56:13,567 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:13,568 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7752): Opening region: {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:56:13,568 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:13,568 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:56:13,568 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7794): checking encryption for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:13,568 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7797): checking classloading for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:13,571 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:13,572 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:56:13,573 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bd4185211d8fb1e40f0aca1aa4a4576 columnFamilyName A 2024-11-28T02:56:13,573 DEBUG [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:13,574 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.HStore(327): Store=3bd4185211d8fb1e40f0aca1aa4a4576/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:56:13,574 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:13,576 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:56:13,576 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bd4185211d8fb1e40f0aca1aa4a4576 columnFamilyName B 2024-11-28T02:56:13,576 DEBUG [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:13,577 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.HStore(327): Store=3bd4185211d8fb1e40f0aca1aa4a4576/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:56:13,577 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:13,578 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:56:13,579 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bd4185211d8fb1e40f0aca1aa4a4576 columnFamilyName C 2024-11-28T02:56:13,579 DEBUG [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:13,579 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.HStore(327): Store=3bd4185211d8fb1e40f0aca1aa4a4576/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:56:13,580 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:13,580 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1038): replaying wal for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:13,581 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:13,582 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:13,583 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1048): stopping wal replay for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:13,583 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1060): Cleaning up temporary data for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:13,584 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T02:56:13,585 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1093): writing seq id for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:13,587 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:56:13,587 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1114): Opened 3bd4185211d8fb1e40f0aca1aa4a4576; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64512303, jitterRate=-0.038691774010658264}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T02:56:13,587 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:13,588 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1006): Region open journal for 3bd4185211d8fb1e40f0aca1aa4a4576: Running coprocessor pre-open hook at 1732762573569Writing region info on filesystem at 1732762573569Initializing all the Stores at 1732762573570 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762573570Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762573570Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762573570Cleaning up temporary data from old regions at 1732762573583 (+13 ms)Running coprocessor post-open hooks at 1732762573587 (+4 ms)Region opened successfully at 1732762573588 (+1 ms) 2024-11-28T02:56:13,588 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., pid=150, masterSystemTime=1732762573562 2024-11-28T02:56:13,590 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:13,590 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:13,591 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=3bd4185211d8fb1e40f0aca1aa4a4576, regionState=OPEN, openSeqNum=2, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:56:13,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:56:13,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-28T02:56:13,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 in 185 msec 2024-11-28T02:56:13,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-28T02:56:13,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, ASSIGN in 345 msec 2024-11-28T02:56:13,596 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T02:56:13,597 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762573596"}]},"ts":"1732762573596"} 2024-11-28T02:56:13,597 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T02:56:13,598 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T02:56:13,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-28T02:56:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-28T02:56:14,576 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:14,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:14,578 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:56:14,578 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:56:14,579 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:56:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:56:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:56:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3117b36c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:56:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:56:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:14,582 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:56:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e4a2275, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:14,584 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:56:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:56:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:56:14,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:56:14,588 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T02:56:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 978bd30859c3,32823,1732762419008 2024-11-28T02:56:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4f35d68b 2024-11-28T02:56:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T02:56:14,591 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T02:56:14,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T02:56:14,593 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:56:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:14,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742362_1538 (size=1097) 2024-11-28T02:56:15,008 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T02:56:15,008 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T02:56:15,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T02:56:15,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, REOPEN/MOVE}] 2024-11-28T02:56:15,017 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, REOPEN/MOVE 2024-11-28T02:56:15,019 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=3bd4185211d8fb1e40f0aca1aa4a4576, regionState=CLOSING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, REOPEN/MOVE because future has completed 2024-11-28T02:56:15,021 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T02:56:15,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:56:15,175 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,175 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T02:56:15,175 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing 3bd4185211d8fb1e40f0aca1aa4a4576, disabling compactions & flushes 2024-11-28T02:56:15,175 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:15,175 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:15,175 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. after waiting 0 ms 2024-11-28T02:56:15,175 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:15,182 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T02:56:15,183 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:15,184 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for 3bd4185211d8fb1e40f0aca1aa4a4576: Waiting for close lock at 1732762575175Running coprocessor pre-close hooks at 1732762575175Disabling compacts and flushes for region at 1732762575175Disabling writes for close at 1732762575175Writing region close event to WAL at 1732762575176 (+1 ms)Running coprocessor post-close hooks at 1732762575183 (+7 ms)Closed at 1732762575183 2024-11-28T02:56:15,184 WARN [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionServer(3299): Not adding moved region record: 3bd4185211d8fb1e40f0aca1aa4a4576 to self. 2024-11-28T02:56:15,186 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,187 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=3bd4185211d8fb1e40f0aca1aa4a4576, regionState=CLOSED 2024-11-28T02:56:15,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:56:15,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-28T02:56:15,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure 3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 in 170 msec 2024-11-28T02:56:15,196 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, REOPEN/MOVE; state=CLOSED, location=978bd30859c3,41631,1732762419946; forceNewPlan=false, retain=true 2024-11-28T02:56:15,347 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=3bd4185211d8fb1e40f0aca1aa4a4576, regionState=OPENING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, REOPEN/MOVE because future has completed 2024-11-28T02:56:15,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:56:15,509 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:15,510 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7752): Opening region: {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:56:15,510 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,511 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:56:15,511 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7794): checking encryption for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,511 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7797): checking classloading for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,514 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,515 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:56:15,515 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bd4185211d8fb1e40f0aca1aa4a4576 columnFamilyName A 2024-11-28T02:56:15,516 DEBUG [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:15,516 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.HStore(327): Store=3bd4185211d8fb1e40f0aca1aa4a4576/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:56:15,517 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,517 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:56:15,518 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bd4185211d8fb1e40f0aca1aa4a4576 columnFamilyName B 2024-11-28T02:56:15,518 DEBUG [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:15,518 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.HStore(327): Store=3bd4185211d8fb1e40f0aca1aa4a4576/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:56:15,518 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,519 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T02:56:15,519 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bd4185211d8fb1e40f0aca1aa4a4576 columnFamilyName C 2024-11-28T02:56:15,519 DEBUG [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:15,519 INFO [StoreOpener-3bd4185211d8fb1e40f0aca1aa4a4576-1 {}] regionserver.HStore(327): Store=3bd4185211d8fb1e40f0aca1aa4a4576/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:56:15,520 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:15,520 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1038): replaying wal for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,520 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,522 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,523 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1048): stopping wal replay for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,523 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1060): Cleaning up temporary data for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,523 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T02:56:15,525 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1093): writing seq id for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,525 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1114): Opened 3bd4185211d8fb1e40f0aca1aa4a4576; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71556170, jitterRate=0.06627002358436584}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T02:56:15,525 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,526 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1006): Region open journal for 3bd4185211d8fb1e40f0aca1aa4a4576: Running coprocessor pre-open hook at 1732762575511Writing region info on filesystem at 1732762575511Initializing all the Stores at 1732762575513 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762575513Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762575513Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732762575513Cleaning up temporary data from old regions at 1732762575523 (+10 ms)Running coprocessor post-open hooks at 1732762575525 (+2 ms)Region opened successfully at 1732762575526 (+1 ms) 2024-11-28T02:56:15,527 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., pid=155, masterSystemTime=1732762575502 2024-11-28T02:56:15,529 DEBUG [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:15,529 INFO [RS_OPEN_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:15,529 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=3bd4185211d8fb1e40f0aca1aa4a4576, regionState=OPEN, openSeqNum=5, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:56:15,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=153 2024-11-28T02:56:15,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure 3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 in 183 msec 2024-11-28T02:56:15,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-28T02:56:15,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, REOPEN/MOVE in 518 msec 2024-11-28T02:56:15,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-28T02:56:15,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 524 msec 2024-11-28T02:56:15,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 946 msec 2024-11-28T02:56:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-28T02:56:15,543 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:15,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:56:15,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:56:15,545 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:56:15,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:56:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:56:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bf0069, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:56:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:56:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:15,547 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:56:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68a2a7f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:56:15,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:56:15,550 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:56:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:56:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:56:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4104af03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:56:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:56:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:15,552 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:56:15,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d3e7804, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:56:15,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:56:15,554 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:56:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:56:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:56:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3140d378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:56:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:56:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:15,555 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:56:15,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c3ddf90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:56:15,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:56:15,557 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:56:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:56:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:56:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3247da14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:56:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:56:15,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:15,558 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:56:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f4374e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,559 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:56:15,559 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:56:15,560 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:56:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:56:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:56:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b7092a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:56:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:56:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:15,561 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:56:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27518c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,562 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:56:15,562 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:56:15,562 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:56:15,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:56:15,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:56:15,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eca3fc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:56:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:56:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:15,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:56:15,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1782d2f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:56:15,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:56:15,565 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:56:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:56:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:56:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5bb66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:56:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:56:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:15,566 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:56:15,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2efdb53f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:56:15,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:56:15,569 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:56:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:56:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:56:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72138c62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:56:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:56:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:15,570 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:56:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eec861d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:56:15,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:56:15,572 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:56:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:56:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:56:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5028a23f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:56:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:56:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:15,573 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:56:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492ce54c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 978bd30859c3,32823,-1 for getting cluster id 2024-11-28T02:56:15,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:56:15,575 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e36574f-6bcb-448f-b5da-ed8f9252fc52' 2024-11-28T02:56:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:56:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e36574f-6bcb-448f-b5da-ed8f9252fc52" 2024-11-28T02:56:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e16800a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [978bd30859c3,32823,-1] 2024-11-28T02:56:15,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:56:15,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:15,577 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:56:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@223b515, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T02:56:15,578 DEBUG [Thread-2419 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:56:15,578 DEBUG [Thread-2423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:56:15,578 DEBUG [Thread-2404 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:56:15,578 DEBUG [Thread-2417 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:56:15,578 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:56:15,578 DEBUG [Thread-2408 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:56:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:56:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:56:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:56:15,579 DEBUG [Thread-2412 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:56:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:56:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:56:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:56:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:56:15,579 DEBUG [Thread-2415 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:56:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-28T02:56:15,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:56:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:56:15,580 DEBUG [Thread-2421 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:56:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:56:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:56:15,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:56:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T02:56:15,581 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:56:15,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:56:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:56:15,581 DEBUG [Thread-2406 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:56:15,581 DEBUG [Thread-2410 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:56:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:56:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:56:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:56:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:56:15,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:56:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:56:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=978bd30859c3,41631,1732762419946, seqNum=-1] 2024-11-28T02:56:15,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:56:15,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:56:15,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:56:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:56:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:56:15,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:56:15,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:56:15,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:56:15,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:56:15,583 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:56:15,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:56:15,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:56:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:56:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:56:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:56:15,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:56:15,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:56:15,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:56:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:56:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:56:15,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5] 2024-11-28T02:56:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:15,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T02:56:15,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:15,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:15,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:15,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:15,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:15,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:15,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112897553c97673c4352aa947d9000ffc659_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762575590/Put/seqid=0 2024-11-28T02:56:15,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742363_1539 (size=12154) 2024-11-28T02:56:15,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762635632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762575595, max timestamp=1732762575637Num uniques 859; 2024-11-28T02:56:15,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:56:15,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762575595, max timestamp=1732762575637Num uniques 859; 2024-11-28T02:56:15,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732762575595, max timestamp=1732762575637Num uniques 859; 2024-11-28T02:56:15,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:56:15,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/978bd30859c3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T02:56:15,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762635656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762635656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762635657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762635657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T02:56:15,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T02:56:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:15,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:15,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762635746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762635765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762635766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762635766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762635766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T02:56:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:15,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:15,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:15,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:15,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T02:56:15,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762635955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762635976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762635976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762635976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762635976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,016 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:16,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112897553c97673c4352aa947d9000ffc659_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112897553c97673c4352aa947d9000ffc659_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:16,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/51b6ebc3609b48f2813b1860b21f7617, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:16,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/51b6ebc3609b48f2813b1860b21f7617 is 175, key is test_row_0/A:col10/1732762575590/Put/seqid=0 2024-11-28T02:56:16,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742364_1540 (size=30955) 2024-11-28T02:56:16,022 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/51b6ebc3609b48f2813b1860b21f7617 2024-11-28T02:56:16,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/84908fb9201b4520bf45651ff1b4e79f is 50, key is test_row_0/B:col10/1732762575590/Put/seqid=0 2024-11-28T02:56:16,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T02:56:16,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:16,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:16,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:16,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:16,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:16,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:16,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742365_1541 (size=12001) 2024-11-28T02:56:16,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/84908fb9201b4520bf45651ff1b4e79f 2024-11-28T02:56:16,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/888c8cd5dffa4aee8f28667a380d4eab is 50, key is test_row_0/C:col10/1732762575590/Put/seqid=0 2024-11-28T02:56:16,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742366_1542 (size=12001) 2024-11-28T02:56:16,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T02:56:16,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:16,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:16,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:16,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:16,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T02:56:16,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762636265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762636286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762636286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762636286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762636286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T02:56:16,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:16,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:16,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:16,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:16,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:16,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/888c8cd5dffa4aee8f28667a380d4eab 2024-11-28T02:56:16,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/51b6ebc3609b48f2813b1860b21f7617 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/51b6ebc3609b48f2813b1860b21f7617 2024-11-28T02:56:16,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/51b6ebc3609b48f2813b1860b21f7617, entries=150, sequenceid=15, filesize=30.2 K 2024-11-28T02:56:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/84908fb9201b4520bf45651ff1b4e79f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/84908fb9201b4520bf45651ff1b4e79f 2024-11-28T02:56:16,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/84908fb9201b4520bf45651ff1b4e79f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T02:56:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/888c8cd5dffa4aee8f28667a380d4eab as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/888c8cd5dffa4aee8f28667a380d4eab 2024-11-28T02:56:16,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/888c8cd5dffa4aee8f28667a380d4eab, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T02:56:16,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 892ms, sequenceid=15, compaction requested=false 2024-11-28T02:56:16,483 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T02:56:16,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:16,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T02:56:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:16,499 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-28T02:56:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:16,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T02:56:16,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128cc9c32a2ae564b0e8ced78124d27af29_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762575655/Put/seqid=0 2024-11-28T02:56:16,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742367_1543 (size=12154) 2024-11-28T02:56:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T02:56:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:16,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:16,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762636797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762636800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762636800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762636801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762636801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762636906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762636916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762636916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762636916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762636916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:16,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:16,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:16,931 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128cc9c32a2ae564b0e8ced78124d27af29_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128cc9c32a2ae564b0e8ced78124d27af29_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/cfb8373de3464bbca86fbc26f67c99e3, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:16,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/cfb8373de3464bbca86fbc26f67c99e3 is 175, key is test_row_0/A:col10/1732762575655/Put/seqid=0 2024-11-28T02:56:16,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742368_1544 (size=30955) 2024-11-28T02:56:17,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762637116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762637126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762637126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762637126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762637126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,336 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/cfb8373de3464bbca86fbc26f67c99e3 2024-11-28T02:56:17,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/08949dd3406c4ea7b374a47b2f25d4d4 is 50, key is test_row_0/B:col10/1732762575655/Put/seqid=0 2024-11-28T02:56:17,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742369_1545 (size=12001) 2024-11-28T02:56:17,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762637425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762637436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762637436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762637436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762637436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T02:56:17,747 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/08949dd3406c4ea7b374a47b2f25d4d4 2024-11-28T02:56:17,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/9392a9f97a9743188d5a36a65168f383 is 50, key is test_row_0/C:col10/1732762575655/Put/seqid=0 2024-11-28T02:56:17,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742370_1546 (size=12001) 2024-11-28T02:56:17,779 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T02:56:17,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762637936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762637946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762637946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762637946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762637946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:18,163 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/9392a9f97a9743188d5a36a65168f383 2024-11-28T02:56:18,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/cfb8373de3464bbca86fbc26f67c99e3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/cfb8373de3464bbca86fbc26f67c99e3 2024-11-28T02:56:18,176 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/cfb8373de3464bbca86fbc26f67c99e3, entries=150, sequenceid=49, filesize=30.2 K 2024-11-28T02:56:18,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/08949dd3406c4ea7b374a47b2f25d4d4 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/08949dd3406c4ea7b374a47b2f25d4d4 2024-11-28T02:56:18,180 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/08949dd3406c4ea7b374a47b2f25d4d4, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T02:56:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/9392a9f97a9743188d5a36a65168f383 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/9392a9f97a9743188d5a36a65168f383 2024-11-28T02:56:18,183 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/9392a9f97a9743188d5a36a65168f383, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T02:56:18,184 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 1684ms, sequenceid=49, compaction requested=false 2024-11-28T02:56:18,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:18,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:18,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-28T02:56:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-28T02:56:18,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-28T02:56:18,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6020 sec 2024-11-28T02:56:18,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.6080 sec 2024-11-28T02:56:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:18,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T02:56:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:18,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128281b7f3227754e3ca93f356fcfa884d0_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762578958/Put/seqid=0 2024-11-28T02:56:18,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742371_1547 (size=12154) 2024-11-28T02:56:18,968 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:18,970 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128281b7f3227754e3ca93f356fcfa884d0_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128281b7f3227754e3ca93f356fcfa884d0_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:18,971 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/56832ce88bef411ea7835026f7ef7db1, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:18,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/56832ce88bef411ea7835026f7ef7db1 is 175, key is test_row_0/A:col10/1732762578958/Put/seqid=0 2024-11-28T02:56:18,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742372_1548 (size=30955) 2024-11-28T02:56:18,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762638974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:18,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762638976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:18,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762638978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:18,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762638978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:18,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762638979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762639086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762639086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762639086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762639086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762639087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762639296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762639296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762639296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762639297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762639297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,375 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/56832ce88bef411ea7835026f7ef7db1 2024-11-28T02:56:19,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/ada3e02495544f8cb67981470c12eb7d is 50, key is test_row_0/B:col10/1732762578958/Put/seqid=0 2024-11-28T02:56:19,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742373_1549 (size=12001) 2024-11-28T02:56:19,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762639606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762639606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762639606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762639606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762639606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:19,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T02:56:19,746 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:19,746 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:56:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-28T02:56:19,748 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:56:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T02:56:19,748 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:56:19,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:56:19,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/ada3e02495544f8cb67981470c12eb7d 2024-11-28T02:56:19,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/b7af8ec4a4c9480dab506ba43e41f718 is 50, key is test_row_0/C:col10/1732762578958/Put/seqid=0 2024-11-28T02:56:19,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742374_1550 (size=12001) 2024-11-28T02:56:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T02:56:19,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-28T02:56:19,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:19,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:19,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:19,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:19,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:20,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-28T02:56:20,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:20,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:20,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:20,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:20,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T02:56:20,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762640116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:20,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762640116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:20,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762640116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:20,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762640116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:20,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762640116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:20,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/b7af8ec4a4c9480dab506ba43e41f718 2024-11-28T02:56:20,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/56832ce88bef411ea7835026f7ef7db1 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/56832ce88bef411ea7835026f7ef7db1 2024-11-28T02:56:20,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/56832ce88bef411ea7835026f7ef7db1, entries=150, sequenceid=63, filesize=30.2 K 2024-11-28T02:56:20,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/ada3e02495544f8cb67981470c12eb7d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/ada3e02495544f8cb67981470c12eb7d 2024-11-28T02:56:20,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/ada3e02495544f8cb67981470c12eb7d, entries=150, sequenceid=63, filesize=11.7 K 2024-11-28T02:56:20,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/b7af8ec4a4c9480dab506ba43e41f718 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b7af8ec4a4c9480dab506ba43e41f718 2024-11-28T02:56:20,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b7af8ec4a4c9480dab506ba43e41f718, entries=150, sequenceid=63, filesize=11.7 K 2024-11-28T02:56:20,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 1246ms, sequenceid=63, compaction requested=true 2024-11-28T02:56:20,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:20,204 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:20,205 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:20,205 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/A is initiating minor compaction (all files) 2024-11-28T02:56:20,205 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/A in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:20,205 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/51b6ebc3609b48f2813b1860b21f7617, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/cfb8373de3464bbca86fbc26f67c99e3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/56832ce88bef411ea7835026f7ef7db1] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=90.7 K 2024-11-28T02:56:20,205 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:20,205 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/51b6ebc3609b48f2813b1860b21f7617, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/cfb8373de3464bbca86fbc26f67c99e3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/56832ce88bef411ea7835026f7ef7db1] 2024-11-28T02:56:20,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-28T02:56:20,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:56:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:20,205 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T02:56:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:56:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:20,205 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:56:20,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:20,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:20,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:20,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:20,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:20,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:20,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:20,206 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51b6ebc3609b48f2813b1860b21f7617, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732762575589 2024-11-28T02:56:20,207 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfb8373de3464bbca86fbc26f67c99e3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732762575655 2024-11-28T02:56:20,207 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:20,207 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/B is initiating minor compaction (all files) 2024-11-28T02:56:20,207 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/B in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:20,207 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/84908fb9201b4520bf45651ff1b4e79f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/08949dd3406c4ea7b374a47b2f25d4d4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/ada3e02495544f8cb67981470c12eb7d] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=35.2 K 2024-11-28T02:56:20,207 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 84908fb9201b4520bf45651ff1b4e79f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732762575589 2024-11-28T02:56:20,207 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56832ce88bef411ea7835026f7ef7db1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732762576800 2024-11-28T02:56:20,208 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 08949dd3406c4ea7b374a47b2f25d4d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732762575655 2024-11-28T02:56:20,208 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting ada3e02495544f8cb67981470c12eb7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732762576800 2024-11-28T02:56:20,223 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:20,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128252714d48dff414380d3494b533e758d_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762578978/Put/seqid=0 2024-11-28T02:56:20,224 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#B#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:20,225 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/1a5ce2df591141c0828b93c600414492 is 50, key is test_row_0/B:col10/1732762578958/Put/seqid=0 2024-11-28T02:56:20,227 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128127a9bc7a7da436d8c5082a01ad62316_3bd4185211d8fb1e40f0aca1aa4a4576 store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:20,229 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128127a9bc7a7da436d8c5082a01ad62316_3bd4185211d8fb1e40f0aca1aa4a4576, store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:20,229 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128127a9bc7a7da436d8c5082a01ad62316_3bd4185211d8fb1e40f0aca1aa4a4576 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:20,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742376_1552 (size=12104) 2024-11-28T02:56:20,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742377_1553 (size=4469) 2024-11-28T02:56:20,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742375_1551 (size=12154) 2024-11-28T02:56:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T02:56:20,639 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/1a5ce2df591141c0828b93c600414492 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/1a5ce2df591141c0828b93c600414492 2024-11-28T02:56:20,642 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/B of 3bd4185211d8fb1e40f0aca1aa4a4576 into 1a5ce2df591141c0828b93c600414492(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:20,642 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:20,642 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/B, priority=13, startTime=1732762580205; duration=0sec 2024-11-28T02:56:20,643 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:20,643 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:B 2024-11-28T02:56:20,643 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:20,643 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:20,643 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/C is initiating minor compaction (all files) 2024-11-28T02:56:20,643 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/C in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:20,643 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/888c8cd5dffa4aee8f28667a380d4eab, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/9392a9f97a9743188d5a36a65168f383, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b7af8ec4a4c9480dab506ba43e41f718] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=35.2 K 2024-11-28T02:56:20,644 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 888c8cd5dffa4aee8f28667a380d4eab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732762575589 2024-11-28T02:56:20,644 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9392a9f97a9743188d5a36a65168f383, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732762575655 2024-11-28T02:56:20,644 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting b7af8ec4a4c9480dab506ba43e41f718, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732762576800 2024-11-28T02:56:20,646 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#A#compaction#467 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:20,646 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/2a2e10fe641e41f18a6110984eadc552 is 175, key is test_row_0/A:col10/1732762578958/Put/seqid=0 2024-11-28T02:56:20,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742378_1554 (size=31058) 2024-11-28T02:56:20,651 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#C#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:20,652 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/d580d0b289b54f2191faee71720267c3 is 50, key is test_row_0/C:col10/1732762578958/Put/seqid=0 2024-11-28T02:56:20,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:20,658 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/2a2e10fe641e41f18a6110984eadc552 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/2a2e10fe641e41f18a6110984eadc552 2024-11-28T02:56:20,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742379_1555 (size=12104) 2024-11-28T02:56:20,663 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128252714d48dff414380d3494b533e758d_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128252714d48dff414380d3494b533e758d_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:20,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/5a8cf96d561a4f14b7e9da2f7e66e26a, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:20,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/5a8cf96d561a4f14b7e9da2f7e66e26a is 175, key is test_row_0/A:col10/1732762578978/Put/seqid=0 2024-11-28T02:56:20,667 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/A of 3bd4185211d8fb1e40f0aca1aa4a4576 into 2a2e10fe641e41f18a6110984eadc552(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:20,667 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:20,667 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/A, priority=13, startTime=1732762580204; duration=0sec 2024-11-28T02:56:20,667 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:20,667 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:A 2024-11-28T02:56:20,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742380_1556 (size=30955) 2024-11-28T02:56:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T02:56:21,062 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/d580d0b289b54f2191faee71720267c3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/d580d0b289b54f2191faee71720267c3 2024-11-28T02:56:21,065 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/C of 3bd4185211d8fb1e40f0aca1aa4a4576 into d580d0b289b54f2191faee71720267c3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:21,065 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:21,065 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/C, priority=13, startTime=1732762580205; duration=0sec 2024-11-28T02:56:21,065 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:21,065 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:C 2024-11-28T02:56:21,072 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/5a8cf96d561a4f14b7e9da2f7e66e26a 2024-11-28T02:56:21,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/4f8cb9438503414db74c17e3ec35adbd is 50, key is test_row_0/B:col10/1732762578978/Put/seqid=0 2024-11-28T02:56:21,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742381_1557 (size=12001) 2024-11-28T02:56:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:21,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:21,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762641164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762641164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762641165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762641170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762641170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762641276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762641276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762641276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762641286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762641286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,481 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/4f8cb9438503414db74c17e3ec35adbd 2024-11-28T02:56:21,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/9abec83a915d493097de3aec9f3d482f is 50, key is test_row_0/C:col10/1732762578978/Put/seqid=0 2024-11-28T02:56:21,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762641486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762641486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762641486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742382_1558 (size=12001) 2024-11-28T02:56:21,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762641495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762641495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762641796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762641796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762641796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762641806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762641806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:21,891 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/9abec83a915d493097de3aec9f3d482f 2024-11-28T02:56:21,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/5a8cf96d561a4f14b7e9da2f7e66e26a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/5a8cf96d561a4f14b7e9da2f7e66e26a 2024-11-28T02:56:21,898 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/5a8cf96d561a4f14b7e9da2f7e66e26a, entries=150, sequenceid=81, filesize=30.2 K 2024-11-28T02:56:21,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/4f8cb9438503414db74c17e3ec35adbd as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/4f8cb9438503414db74c17e3ec35adbd 2024-11-28T02:56:21,901 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/4f8cb9438503414db74c17e3ec35adbd, entries=150, sequenceid=81, filesize=11.7 K 2024-11-28T02:56:21,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/9abec83a915d493097de3aec9f3d482f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/9abec83a915d493097de3aec9f3d482f 2024-11-28T02:56:21,904 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/9abec83a915d493097de3aec9f3d482f, entries=150, sequenceid=81, filesize=11.7 K 2024-11-28T02:56:21,905 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 1700ms, sequenceid=81, compaction requested=false 2024-11-28T02:56:21,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:21,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:21,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-28T02:56:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-28T02:56:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T02:56:21,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-28T02:56:21,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1580 sec 2024-11-28T02:56:21,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.1620 sec 2024-11-28T02:56:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:22,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T02:56:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:22,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128db5d2f6aec4f44399b9823a374657ed5_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762582307/Put/seqid=0 2024-11-28T02:56:22,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742383_1559 (size=14594) 2024-11-28T02:56:22,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762642321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762642322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762642323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762642324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762642326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762642435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762642436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762642436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762642436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762642436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762642645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762642646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762642646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762642646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762642646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,717 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:22,720 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128db5d2f6aec4f44399b9823a374657ed5_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128db5d2f6aec4f44399b9823a374657ed5_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:22,721 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/478fea258e824c138b7c55e28d26bbb0, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:22,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/478fea258e824c138b7c55e28d26bbb0 is 175, key is test_row_0/A:col10/1732762582307/Put/seqid=0 2024-11-28T02:56:22,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742384_1560 (size=39549) 2024-11-28T02:56:22,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762642956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762642956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762642956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762642957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762642957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:23,124 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/478fea258e824c138b7c55e28d26bbb0 2024-11-28T02:56:23,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/adc91eed5056408893ba40a6746dc9d1 is 50, key is test_row_0/B:col10/1732762582307/Put/seqid=0 2024-11-28T02:56:23,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742385_1561 (size=12001) 2024-11-28T02:56:23,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762643465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:23,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762643465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:23,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762643466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:23,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762643466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:23,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762643466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:23,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/adc91eed5056408893ba40a6746dc9d1 2024-11-28T02:56:23,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/358455a6c21b4baca3dee56a137cc4cf is 50, key is test_row_0/C:col10/1732762582307/Put/seqid=0 2024-11-28T02:56:23,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742386_1562 (size=12001) 2024-11-28T02:56:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T02:56:23,916 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:23,917 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:56:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-28T02:56:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T02:56:23,918 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:56:23,918 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:56:23,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:56:23,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/358455a6c21b4baca3dee56a137cc4cf 2024-11-28T02:56:23,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/478fea258e824c138b7c55e28d26bbb0 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/478fea258e824c138b7c55e28d26bbb0 2024-11-28T02:56:23,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/478fea258e824c138b7c55e28d26bbb0, entries=200, sequenceid=100, filesize=38.6 K 2024-11-28T02:56:23,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/adc91eed5056408893ba40a6746dc9d1 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/adc91eed5056408893ba40a6746dc9d1 2024-11-28T02:56:23,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/adc91eed5056408893ba40a6746dc9d1, entries=150, sequenceid=100, filesize=11.7 K 2024-11-28T02:56:23,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/358455a6c21b4baca3dee56a137cc4cf as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/358455a6c21b4baca3dee56a137cc4cf 2024-11-28T02:56:23,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/358455a6c21b4baca3dee56a137cc4cf, entries=150, sequenceid=100, filesize=11.7 K 2024-11-28T02:56:23,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 1646ms, sequenceid=100, compaction requested=true 2024-11-28T02:56:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:56:23,954 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:56:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:56:23,954 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:23,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:23,956 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:23,956 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:23,956 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/A is initiating minor compaction (all files) 2024-11-28T02:56:23,956 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/B is initiating minor compaction (all files) 2024-11-28T02:56:23,956 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/A in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:23,956 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/B in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:23,956 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/2a2e10fe641e41f18a6110984eadc552, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/5a8cf96d561a4f14b7e9da2f7e66e26a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/478fea258e824c138b7c55e28d26bbb0] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=99.2 K 2024-11-28T02:56:23,956 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/1a5ce2df591141c0828b93c600414492, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/4f8cb9438503414db74c17e3ec35adbd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/adc91eed5056408893ba40a6746dc9d1] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=35.3 K 2024-11-28T02:56:23,956 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:23,956 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/2a2e10fe641e41f18a6110984eadc552, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/5a8cf96d561a4f14b7e9da2f7e66e26a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/478fea258e824c138b7c55e28d26bbb0] 2024-11-28T02:56:23,956 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a5ce2df591141c0828b93c600414492, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732762576800 2024-11-28T02:56:23,956 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a2e10fe641e41f18a6110984eadc552, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732762576800 2024-11-28T02:56:23,957 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a8cf96d561a4f14b7e9da2f7e66e26a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732762578972 2024-11-28T02:56:23,957 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f8cb9438503414db74c17e3ec35adbd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732762578972 2024-11-28T02:56:23,957 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 478fea258e824c138b7c55e28d26bbb0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732762581164 2024-11-28T02:56:23,957 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting adc91eed5056408893ba40a6746dc9d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732762581164 2024-11-28T02:56:23,965 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:23,971 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128d50490b05f2046a9996102bdd1d7bea9_3bd4185211d8fb1e40f0aca1aa4a4576 store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:23,972 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#B#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:23,973 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/5b558bd92d2742a19d73b50b29c737df is 50, key is test_row_0/B:col10/1732762582307/Put/seqid=0 2024-11-28T02:56:23,973 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128d50490b05f2046a9996102bdd1d7bea9_3bd4185211d8fb1e40f0aca1aa4a4576, store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:23,973 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d50490b05f2046a9996102bdd1d7bea9_3bd4185211d8fb1e40f0aca1aa4a4576 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:23,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742387_1563 (size=12207) 2024-11-28T02:56:23,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742388_1564 (size=4469) 2024-11-28T02:56:23,987 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/5b558bd92d2742a19d73b50b29c737df as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/5b558bd92d2742a19d73b50b29c737df 2024-11-28T02:56:23,990 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/B of 3bd4185211d8fb1e40f0aca1aa4a4576 into 5b558bd92d2742a19d73b50b29c737df(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:23,990 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:23,990 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/B, priority=13, startTime=1732762583954; duration=0sec 2024-11-28T02:56:23,990 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:23,990 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:B 2024-11-28T02:56:23,990 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:23,991 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:23,991 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/C is initiating minor compaction (all files) 2024-11-28T02:56:23,991 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/C in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:23,991 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/d580d0b289b54f2191faee71720267c3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/9abec83a915d493097de3aec9f3d482f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/358455a6c21b4baca3dee56a137cc4cf] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=35.3 K 2024-11-28T02:56:23,992 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting d580d0b289b54f2191faee71720267c3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732762576800 2024-11-28T02:56:23,992 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9abec83a915d493097de3aec9f3d482f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732762578972 2024-11-28T02:56:23,992 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 358455a6c21b4baca3dee56a137cc4cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732762581164 2024-11-28T02:56:24,001 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:24,002 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/bf87fbba44304f91a2cb8933d53fb507 is 50, key is test_row_0/C:col10/1732762582307/Put/seqid=0 2024-11-28T02:56:24,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742389_1565 (size=12207) 2024-11-28T02:56:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T02:56:24,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-28T02:56:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:24,071 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T02:56:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:24,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282c4e503b1c9a47f9890d38440bbaf4e1_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762582323/Put/seqid=0 2024-11-28T02:56:24,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742390_1566 (size=12154) 2024-11-28T02:56:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T02:56:24,385 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#A#compaction#474 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:24,385 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/f33fe37c40004089907622ebaa552de7 is 175, key is test_row_0/A:col10/1732762582307/Put/seqid=0 2024-11-28T02:56:24,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742391_1567 (size=31161) 2024-11-28T02:56:24,409 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/bf87fbba44304f91a2cb8933d53fb507 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/bf87fbba44304f91a2cb8933d53fb507 2024-11-28T02:56:24,413 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/C of 3bd4185211d8fb1e40f0aca1aa4a4576 into bf87fbba44304f91a2cb8933d53fb507(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:24,413 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:24,413 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/C, priority=13, startTime=1732762583954; duration=0sec 2024-11-28T02:56:24,413 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:24,413 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:C 2024-11-28T02:56:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:24,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:24,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:24,483 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282c4e503b1c9a47f9890d38440bbaf4e1_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282c4e503b1c9a47f9890d38440bbaf4e1_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:24,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762644491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762644491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/d8bd689d405049868e2070293622f872, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:24,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762644495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/d8bd689d405049868e2070293622f872 is 175, key is test_row_0/A:col10/1732762582323/Put/seqid=0 2024-11-28T02:56:24,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742392_1568 (size=30955) 2024-11-28T02:56:24,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762644497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762644497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T02:56:24,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762644606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762644606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762644606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762644606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762644606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,791 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/f33fe37c40004089907622ebaa552de7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/f33fe37c40004089907622ebaa552de7 2024-11-28T02:56:24,794 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/A of 3bd4185211d8fb1e40f0aca1aa4a4576 into f33fe37c40004089907622ebaa552de7(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:24,794 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:24,794 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/A, priority=13, startTime=1732762583954; duration=0sec 2024-11-28T02:56:24,794 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:24,795 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:A 2024-11-28T02:56:24,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762644815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762644815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762644816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762644816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762644816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:24,903 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/d8bd689d405049868e2070293622f872 2024-11-28T02:56:24,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/0824323d4e75497690155acb794cd472 is 50, key is test_row_0/B:col10/1732762582323/Put/seqid=0 2024-11-28T02:56:24,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742393_1569 (size=12001) 2024-11-28T02:56:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T02:56:25,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762645126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:25,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:25,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762645126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762645127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:25,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762645127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:25,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762645127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:25,312 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/0824323d4e75497690155acb794cd472 2024-11-28T02:56:25,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/0502d658d4124b0489df736ea870a162 is 50, key is test_row_0/C:col10/1732762582323/Put/seqid=0 2024-11-28T02:56:25,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742394_1570 (size=12001) 2024-11-28T02:56:25,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762645635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762645636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:25,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762645636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:25,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762645636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:25,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762645636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:25,743 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/0502d658d4124b0489df736ea870a162 2024-11-28T02:56:25,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/d8bd689d405049868e2070293622f872 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/d8bd689d405049868e2070293622f872 2024-11-28T02:56:25,749 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/d8bd689d405049868e2070293622f872, entries=150, sequenceid=117, filesize=30.2 K 2024-11-28T02:56:25,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/0824323d4e75497690155acb794cd472 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/0824323d4e75497690155acb794cd472 2024-11-28T02:56:25,753 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/0824323d4e75497690155acb794cd472, entries=150, sequenceid=117, filesize=11.7 K 2024-11-28T02:56:25,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/0502d658d4124b0489df736ea870a162 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/0502d658d4124b0489df736ea870a162 2024-11-28T02:56:25,756 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/0502d658d4124b0489df736ea870a162, entries=150, sequenceid=117, filesize=11.7 K 2024-11-28T02:56:25,756 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 1685ms, sequenceid=117, compaction requested=false 2024-11-28T02:56:25,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-28T02:56:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-28T02:56:25,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-28T02:56:25,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8400 sec 2024-11-28T02:56:25,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8430 sec 2024-11-28T02:56:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T02:56:26,066 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:26,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:56:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-28T02:56:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T02:56:26,068 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:56:26,069 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:56:26,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:56:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T02:56:26,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-28T02:56:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:26,221 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T02:56:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:26,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128610eff1d93b0483985a444251965eee2_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762584496/Put/seqid=0 2024-11-28T02:56:26,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742395_1571 (size=12304) 2024-11-28T02:56:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:26,231 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128610eff1d93b0483985a444251965eee2_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128610eff1d93b0483985a444251965eee2_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:26,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/f9e803423f3b432d892ba6e0f8b4f79a, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:26,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/f9e803423f3b432d892ba6e0f8b4f79a is 175, key is test_row_0/A:col10/1732762584496/Put/seqid=0 2024-11-28T02:56:26,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742396_1572 (size=31105) 2024-11-28T02:56:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T02:56:26,636 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/f9e803423f3b432d892ba6e0f8b4f79a 2024-11-28T02:56:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/24fc2e5102fc49849aebd90c59ed7a58 is 50, key is test_row_0/B:col10/1732762584496/Put/seqid=0 2024-11-28T02:56:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:26,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:26,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742397_1573 (size=12151) 2024-11-28T02:56:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T02:56:26,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:26,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762646662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762646662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:26,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762646700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:26,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762646700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:26,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:26,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762646701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:26,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:26,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762646805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:26,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762646806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:26,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762646816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:26,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762646816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:26,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762646816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762647016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762647016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762647026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762647026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762647026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,056 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/24fc2e5102fc49849aebd90c59ed7a58 2024-11-28T02:56:27,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/ba95b50621cf42caae23638a2090f638 is 50, key is test_row_0/C:col10/1732762584496/Put/seqid=0 2024-11-28T02:56:27,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742398_1574 (size=12151) 2024-11-28T02:56:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T02:56:27,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762647325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762647326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762647336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762647336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762647336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,465 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/ba95b50621cf42caae23638a2090f638 2024-11-28T02:56:27,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/f9e803423f3b432d892ba6e0f8b4f79a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/f9e803423f3b432d892ba6e0f8b4f79a 2024-11-28T02:56:27,472 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/f9e803423f3b432d892ba6e0f8b4f79a, entries=150, sequenceid=136, filesize=30.4 K 2024-11-28T02:56:27,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/24fc2e5102fc49849aebd90c59ed7a58 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/24fc2e5102fc49849aebd90c59ed7a58 2024-11-28T02:56:27,475 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/24fc2e5102fc49849aebd90c59ed7a58, entries=150, sequenceid=136, filesize=11.9 K 2024-11-28T02:56:27,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/ba95b50621cf42caae23638a2090f638 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/ba95b50621cf42caae23638a2090f638 2024-11-28T02:56:27,478 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/ba95b50621cf42caae23638a2090f638, entries=150, sequenceid=136, filesize=11.9 K 2024-11-28T02:56:27,479 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 1258ms, sequenceid=136, compaction requested=true 2024-11-28T02:56:27,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:27,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:27,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-28T02:56:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-28T02:56:27,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-28T02:56:27,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-28T02:56:27,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.4160 sec 2024-11-28T02:56:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:27,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T02:56:27,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:27,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:27,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:27,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:27,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:27,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:27,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fb6f64e0be6242628187d0e4685a74e8_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762586673/Put/seqid=0 2024-11-28T02:56:27,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742399_1575 (size=14794) 2024-11-28T02:56:27,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762647850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762647850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762647853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762647854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762647855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762647966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762647966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762647966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762647966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762647966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:27,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:28,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762648176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:28,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762648186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762648186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:28,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762648186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:28,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762648187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T02:56:28,216 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:28,217 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:56:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-28T02:56:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T02:56:28,218 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:56:28,219 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:56:28,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:56:28,248 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:28,250 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fb6f64e0be6242628187d0e4685a74e8_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fb6f64e0be6242628187d0e4685a74e8_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:28,251 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/075fd2776304406baa2edad4494f00cf, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:28,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/075fd2776304406baa2edad4494f00cf is 175, key is test_row_0/A:col10/1732762586673/Put/seqid=0 2024-11-28T02:56:28,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742400_1576 (size=39749) 2024-11-28T02:56:28,255 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/075fd2776304406baa2edad4494f00cf 2024-11-28T02:56:28,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/e150c74768ac42e39d08d524e8565cf1 is 50, key is test_row_0/B:col10/1732762586673/Put/seqid=0 2024-11-28T02:56:28,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742401_1577 (size=12151) 2024-11-28T02:56:28,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/e150c74768ac42e39d08d524e8565cf1 2024-11-28T02:56:28,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/5760b684e0374118a22fea48b1b9fb94 is 50, key is test_row_0/C:col10/1732762586673/Put/seqid=0 2024-11-28T02:56:28,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742402_1578 (size=12151) 2024-11-28T02:56:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T02:56:28,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-28T02:56:28,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:28,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:28,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:28,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:28,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762648486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:28,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762648496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:28,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762648496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:28,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762648496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:28,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762648497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:28,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-28T02:56:28,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:28,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:28,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:28,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:28,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T02:56:28,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/5760b684e0374118a22fea48b1b9fb94 2024-11-28T02:56:28,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/075fd2776304406baa2edad4494f00cf as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/075fd2776304406baa2edad4494f00cf 2024-11-28T02:56:28,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-28T02:56:28,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:28,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:28,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:28,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:28,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:28,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/075fd2776304406baa2edad4494f00cf, entries=200, sequenceid=155, filesize=38.8 K 2024-11-28T02:56:28,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/e150c74768ac42e39d08d524e8565cf1 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/e150c74768ac42e39d08d524e8565cf1 2024-11-28T02:56:28,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/e150c74768ac42e39d08d524e8565cf1, entries=150, sequenceid=155, filesize=11.9 K 2024-11-28T02:56:28,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/5760b684e0374118a22fea48b1b9fb94 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/5760b684e0374118a22fea48b1b9fb94 2024-11-28T02:56:28,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/5760b684e0374118a22fea48b1b9fb94, entries=150, sequenceid=155, filesize=11.9 K 2024-11-28T02:56:28,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 847ms, sequenceid=155, compaction requested=true 2024-11-28T02:56:28,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:28,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:56:28,686 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:56:28,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:28,686 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:56:28,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:56:28,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:28,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:56:28,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:28,694 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:56:28,694 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:56:28,694 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/B is initiating minor compaction (all files) 2024-11-28T02:56:28,694 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/A is initiating minor compaction (all files) 2024-11-28T02:56:28,694 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/B in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:28,694 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/A in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:28,694 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/f33fe37c40004089907622ebaa552de7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/d8bd689d405049868e2070293622f872, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/f9e803423f3b432d892ba6e0f8b4f79a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/075fd2776304406baa2edad4494f00cf] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=129.9 K 2024-11-28T02:56:28,694 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/5b558bd92d2742a19d73b50b29c737df, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/0824323d4e75497690155acb794cd472, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/24fc2e5102fc49849aebd90c59ed7a58, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/e150c74768ac42e39d08d524e8565cf1] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=47.4 K 2024-11-28T02:56:28,694 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:28,695 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/f33fe37c40004089907622ebaa552de7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/d8bd689d405049868e2070293622f872, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/f9e803423f3b432d892ba6e0f8b4f79a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/075fd2776304406baa2edad4494f00cf] 2024-11-28T02:56:28,695 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f33fe37c40004089907622ebaa552de7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732762581164 2024-11-28T02:56:28,695 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b558bd92d2742a19d73b50b29c737df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732762581164 2024-11-28T02:56:28,695 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0824323d4e75497690155acb794cd472, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732762582320 2024-11-28T02:56:28,695 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8bd689d405049868e2070293622f872, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732762582320 2024-11-28T02:56:28,695 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 24fc2e5102fc49849aebd90c59ed7a58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732762584494 2024-11-28T02:56:28,695 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9e803423f3b432d892ba6e0f8b4f79a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732762584494 2024-11-28T02:56:28,695 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting e150c74768ac42e39d08d524e8565cf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732762586668 2024-11-28T02:56:28,695 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 075fd2776304406baa2edad4494f00cf, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732762586668 2024-11-28T02:56:28,701 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:28,703 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#B#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:28,703 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/f15fec348c1249fa8a5a054a03b6da93 is 50, key is test_row_0/B:col10/1732762586673/Put/seqid=0 2024-11-28T02:56:28,714 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128b17b5658a5d245aabe37cef64ac1dd28_3bd4185211d8fb1e40f0aca1aa4a4576 store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:28,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742403_1579 (size=12493) 2024-11-28T02:56:28,716 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128b17b5658a5d245aabe37cef64ac1dd28_3bd4185211d8fb1e40f0aca1aa4a4576, store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:28,716 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b17b5658a5d245aabe37cef64ac1dd28_3bd4185211d8fb1e40f0aca1aa4a4576 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:28,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742404_1580 (size=4469) 2024-11-28T02:56:28,721 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/f15fec348c1249fa8a5a054a03b6da93 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/f15fec348c1249fa8a5a054a03b6da93 2024-11-28T02:56:28,722 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#A#compaction#487 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:28,723 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/2b70bf3b241b40bdaee4e05ad97238ad is 175, key is test_row_0/A:col10/1732762586673/Put/seqid=0 2024-11-28T02:56:28,726 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/B of 3bd4185211d8fb1e40f0aca1aa4a4576 into f15fec348c1249fa8a5a054a03b6da93(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:28,726 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:28,726 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/B, priority=12, startTime=1732762588686; duration=0sec 2024-11-28T02:56:28,726 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:28,726 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:B 2024-11-28T02:56:28,726 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:56:28,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742405_1581 (size=31447) 2024-11-28T02:56:28,728 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:56:28,728 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/C is initiating minor compaction (all files) 2024-11-28T02:56:28,728 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/C in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:28,728 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/bf87fbba44304f91a2cb8933d53fb507, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/0502d658d4124b0489df736ea870a162, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/ba95b50621cf42caae23638a2090f638, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/5760b684e0374118a22fea48b1b9fb94] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=47.4 K 2024-11-28T02:56:28,728 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting bf87fbba44304f91a2cb8933d53fb507, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732762581164 2024-11-28T02:56:28,728 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0502d658d4124b0489df736ea870a162, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732762582320 2024-11-28T02:56:28,728 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting ba95b50621cf42caae23638a2090f638, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732762584494 2024-11-28T02:56:28,729 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5760b684e0374118a22fea48b1b9fb94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732762586668 2024-11-28T02:56:28,731 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/2b70bf3b241b40bdaee4e05ad97238ad as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/2b70bf3b241b40bdaee4e05ad97238ad 2024-11-28T02:56:28,735 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/A of 3bd4185211d8fb1e40f0aca1aa4a4576 into 2b70bf3b241b40bdaee4e05ad97238ad(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:28,735 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:28,735 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/A, priority=12, startTime=1732762588686; duration=0sec 2024-11-28T02:56:28,735 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:28,735 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:A 2024-11-28T02:56:28,737 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#C#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:28,737 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/84e864c6c32b4c1c945e201bb5905b0d is 50, key is test_row_0/C:col10/1732762586673/Put/seqid=0 2024-11-28T02:56:28,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742406_1582 (size=12493) 2024-11-28T02:56:28,744 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/84e864c6c32b4c1c945e201bb5905b0d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/84e864c6c32b4c1c945e201bb5905b0d 2024-11-28T02:56:28,748 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/C of 3bd4185211d8fb1e40f0aca1aa4a4576 into 84e864c6c32b4c1c945e201bb5905b0d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:28,748 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:28,748 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/C, priority=12, startTime=1732762588686; duration=0sec 2024-11-28T02:56:28,748 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:28,748 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:C 2024-11-28T02:56:28,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-28T02:56:28,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:28,829 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T02:56:28,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:28,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:28,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:28,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:28,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:28,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:28,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289756c4fafa0d4ec8a3bc3a909affdcbc_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762587854/Put/seqid=0 2024-11-28T02:56:28,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742407_1583 (size=12304) 2024-11-28T02:56:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T02:56:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:28,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:29,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762649016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762649017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762649018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762649020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762649021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762649126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762649126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762649135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762649135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:29,242 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289756c4fafa0d4ec8a3bc3a909affdcbc_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289756c4fafa0d4ec8a3bc3a909affdcbc_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:29,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/11b5b60ecaca45ec92f6948922fd89af, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:29,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/11b5b60ecaca45ec92f6948922fd89af is 175, key is test_row_0/A:col10/1732762587854/Put/seqid=0 2024-11-28T02:56:29,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742408_1584 (size=31105) 2024-11-28T02:56:29,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762649336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762649336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762649345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762649345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T02:56:29,647 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/11b5b60ecaca45ec92f6948922fd89af 2024-11-28T02:56:29,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762649646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762649646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/d2abd584604f44e7b79a6544d7b3e632 is 50, key is test_row_0/B:col10/1732762587854/Put/seqid=0 2024-11-28T02:56:29,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762649656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762649656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:29,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742409_1585 (size=12151) 2024-11-28T02:56:30,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762650026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:30,029 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 3bd4185211d8fb1e40f0aca1aa4a4576 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:56:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:30,059 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/d2abd584604f44e7b79a6544d7b3e632 2024-11-28T02:56:30,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/812ecd6d071e406d89f8232199ae331f is 50, key is test_row_0/C:col10/1732762587854/Put/seqid=0 2024-11-28T02:56:30,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742410_1586 (size=12151) 2024-11-28T02:56:30,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762650156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:30,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762650156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:30,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762650165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:30,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762650165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T02:56:30,468 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/812ecd6d071e406d89f8232199ae331f 2024-11-28T02:56:30,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/11b5b60ecaca45ec92f6948922fd89af as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/11b5b60ecaca45ec92f6948922fd89af 2024-11-28T02:56:30,473 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/11b5b60ecaca45ec92f6948922fd89af, entries=150, sequenceid=171, filesize=30.4 K 2024-11-28T02:56:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/d2abd584604f44e7b79a6544d7b3e632 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/d2abd584604f44e7b79a6544d7b3e632 2024-11-28T02:56:30,476 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/d2abd584604f44e7b79a6544d7b3e632, entries=150, sequenceid=171, filesize=11.9 K 2024-11-28T02:56:30,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/812ecd6d071e406d89f8232199ae331f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/812ecd6d071e406d89f8232199ae331f 2024-11-28T02:56:30,480 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/812ecd6d071e406d89f8232199ae331f, entries=150, sequenceid=171, filesize=11.9 K 2024-11-28T02:56:30,480 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 1651ms, sequenceid=171, compaction requested=false 2024-11-28T02:56:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-28T02:56:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-28T02:56:30,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-28T02:56:30,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2630 sec 2024-11-28T02:56:30,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.2670 sec 2024-11-28T02:56:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:31,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T02:56:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:31,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bc8733421901475194ec50c88e627ae1_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762591167/Put/seqid=0 2024-11-28T02:56:31,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742411_1587 (size=14794) 2024-11-28T02:56:31,179 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:31,181 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bc8733421901475194ec50c88e627ae1_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bc8733421901475194ec50c88e627ae1_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:31,182 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/08b1465d53b142a7b3483bc5e3ab0d88, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:31,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/08b1465d53b142a7b3483bc5e3ab0d88 is 175, key is test_row_0/A:col10/1732762591167/Put/seqid=0 2024-11-28T02:56:31,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742412_1588 (size=39749) 2024-11-28T02:56:31,185 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/08b1465d53b142a7b3483bc5e3ab0d88 2024-11-28T02:56:31,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762651186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/48f67db3dbd944ddb9d975cc3d158e92 is 50, key is test_row_0/B:col10/1732762591167/Put/seqid=0 2024-11-28T02:56:31,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762651188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762651189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762651189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742413_1589 (size=12151) 2024-11-28T02:56:31,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762651296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762651296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762651306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762651306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762651505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762651505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762651516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762651516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/48f67db3dbd944ddb9d975cc3d158e92 2024-11-28T02:56:31,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/01b63a37765a481d8d98061f62507ad2 is 50, key is test_row_0/C:col10/1732762591167/Put/seqid=0 2024-11-28T02:56:31,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742414_1590 (size=12151) 2024-11-28T02:56:31,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/01b63a37765a481d8d98061f62507ad2 2024-11-28T02:56:31,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/08b1465d53b142a7b3483bc5e3ab0d88 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/08b1465d53b142a7b3483bc5e3ab0d88 2024-11-28T02:56:31,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/08b1465d53b142a7b3483bc5e3ab0d88, entries=200, sequenceid=191, filesize=38.8 K 2024-11-28T02:56:31,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/48f67db3dbd944ddb9d975cc3d158e92 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/48f67db3dbd944ddb9d975cc3d158e92 2024-11-28T02:56:31,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/48f67db3dbd944ddb9d975cc3d158e92, entries=150, sequenceid=191, filesize=11.9 K 2024-11-28T02:56:31,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/01b63a37765a481d8d98061f62507ad2 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/01b63a37765a481d8d98061f62507ad2 2024-11-28T02:56:31,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/01b63a37765a481d8d98061f62507ad2, entries=150, sequenceid=191, filesize=11.9 K 2024-11-28T02:56:31,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 453ms, sequenceid=191, compaction requested=true 2024-11-28T02:56:31,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:31,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:56:31,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:31,622 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:31,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:56:31,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:31,622 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:31,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:56:31,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:31,622 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:31,622 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/B is initiating minor compaction (all files) 2024-11-28T02:56:31,622 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:31,622 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/A is initiating minor compaction (all files) 2024-11-28T02:56:31,622 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/B in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:31,622 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/A in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:31,623 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/f15fec348c1249fa8a5a054a03b6da93, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/d2abd584604f44e7b79a6544d7b3e632, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/48f67db3dbd944ddb9d975cc3d158e92] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=35.9 K 2024-11-28T02:56:31,623 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/2b70bf3b241b40bdaee4e05ad97238ad, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/11b5b60ecaca45ec92f6948922fd89af, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/08b1465d53b142a7b3483bc5e3ab0d88] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=99.9 K 2024-11-28T02:56:31,623 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:31,623 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/2b70bf3b241b40bdaee4e05ad97238ad, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/11b5b60ecaca45ec92f6948922fd89af, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/08b1465d53b142a7b3483bc5e3ab0d88] 2024-11-28T02:56:31,623 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting f15fec348c1249fa8a5a054a03b6da93, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732762586668 2024-11-28T02:56:31,623 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b70bf3b241b40bdaee4e05ad97238ad, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732762586668 2024-11-28T02:56:31,623 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting d2abd584604f44e7b79a6544d7b3e632, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732762587848 2024-11-28T02:56:31,623 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11b5b60ecaca45ec92f6948922fd89af, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732762587848 2024-11-28T02:56:31,623 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 48f67db3dbd944ddb9d975cc3d158e92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732762589016 2024-11-28T02:56:31,623 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08b1465d53b142a7b3483bc5e3ab0d88, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732762589011 2024-11-28T02:56:31,627 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:31,628 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:31,629 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/9f293e870a99435da6e6be1f4372bdf5 is 50, key is test_row_0/B:col10/1732762591167/Put/seqid=0 2024-11-28T02:56:31,629 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128e7db884431744fa3996b4872bf3a6645_3bd4185211d8fb1e40f0aca1aa4a4576 store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:31,631 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128e7db884431744fa3996b4872bf3a6645_3bd4185211d8fb1e40f0aca1aa4a4576, store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:31,631 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e7db884431744fa3996b4872bf3a6645_3bd4185211d8fb1e40f0aca1aa4a4576 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:31,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742415_1591 (size=12595) 2024-11-28T02:56:31,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742416_1592 (size=4469) 2024-11-28T02:56:31,635 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#A#compaction#496 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:31,636 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/61cff6407bdb49dbbcbb678d19095d88 is 175, key is test_row_0/A:col10/1732762591167/Put/seqid=0 2024-11-28T02:56:31,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742417_1593 (size=31549) 2024-11-28T02:56:31,645 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/61cff6407bdb49dbbcbb678d19095d88 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/61cff6407bdb49dbbcbb678d19095d88 2024-11-28T02:56:31,649 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/A of 3bd4185211d8fb1e40f0aca1aa4a4576 into 61cff6407bdb49dbbcbb678d19095d88(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:31,649 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:31,649 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/A, priority=13, startTime=1732762591621; duration=0sec 2024-11-28T02:56:31,649 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:31,649 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:A 2024-11-28T02:56:31,650 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:31,650 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:31,650 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/C is initiating minor compaction (all files) 2024-11-28T02:56:31,650 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/C in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:31,650 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/84e864c6c32b4c1c945e201bb5905b0d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/812ecd6d071e406d89f8232199ae331f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/01b63a37765a481d8d98061f62507ad2] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=35.9 K 2024-11-28T02:56:31,651 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84e864c6c32b4c1c945e201bb5905b0d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732762586668 2024-11-28T02:56:31,651 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 812ecd6d071e406d89f8232199ae331f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732762587848 2024-11-28T02:56:31,651 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01b63a37765a481d8d98061f62507ad2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732762589016 2024-11-28T02:56:31,660 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:31,660 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/72cd1ca01aa94bc39ff525f68bfd482e is 50, key is test_row_0/C:col10/1732762591167/Put/seqid=0 2024-11-28T02:56:31,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742418_1594 (size=12595) 2024-11-28T02:56:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:31,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T02:56:31,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:31,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:31,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:31,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:31,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:31,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:31,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a3230ff09ec24eaeb435685130b0dbd6_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762591816/Put/seqid=0 2024-11-28T02:56:31,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742419_1595 (size=14794) 2024-11-28T02:56:31,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762651840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762651842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762651843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762651844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762651955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762651956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762651956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762651956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,037 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/9f293e870a99435da6e6be1f4372bdf5 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/9f293e870a99435da6e6be1f4372bdf5 2024-11-28T02:56:32,040 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/B of 3bd4185211d8fb1e40f0aca1aa4a4576 into 9f293e870a99435da6e6be1f4372bdf5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:32,040 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:32,040 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/B, priority=13, startTime=1732762591621; duration=0sec 2024-11-28T02:56:32,040 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:32,040 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:B 2024-11-28T02:56:32,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762652046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:32,050 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 3bd4185211d8fb1e40f0aca1aa4a4576 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:56:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,068 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/72cd1ca01aa94bc39ff525f68bfd482e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/72cd1ca01aa94bc39ff525f68bfd482e 2024-11-28T02:56:32,071 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/C of 3bd4185211d8fb1e40f0aca1aa4a4576 into 72cd1ca01aa94bc39ff525f68bfd482e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:32,071 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:32,071 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/C, priority=13, startTime=1732762591622; duration=0sec 2024-11-28T02:56:32,071 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:32,071 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:C 2024-11-28T02:56:32,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762652166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:32,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762652166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762652166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762652167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,226 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:32,228 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a3230ff09ec24eaeb435685130b0dbd6_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a3230ff09ec24eaeb435685130b0dbd6_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:32,229 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/fae6c8cd9716487d88582b4ab9a42d56, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:32,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/fae6c8cd9716487d88582b4ab9a42d56 is 175, key is test_row_0/A:col10/1732762591816/Put/seqid=0 2024-11-28T02:56:32,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742420_1596 (size=39749) 2024-11-28T02:56:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T02:56:32,376 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:32,376 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:56:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-28T02:56:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T02:56:32,378 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:56:32,378 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:56:32,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:56:32,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762652475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:32,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762652475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762652476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:32,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762652476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T02:56:32,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-28T02:56:32,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:32,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:32,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:32,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:32,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:32,632 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/fae6c8cd9716487d88582b4ab9a42d56 2024-11-28T02:56:32,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/04b78abb6d3f4c998449e24ecdad0ca8 is 50, key is test_row_0/B:col10/1732762591816/Put/seqid=0 2024-11-28T02:56:32,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742421_1597 (size=12151) 2024-11-28T02:56:32,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-28T02:56:32,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:32,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:32,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:32,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:32,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T02:56:32,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-28T02:56:32,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:32,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:32,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:32,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:32,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:32,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762652985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762652986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:32,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762652986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:32,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762652986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-28T02:56:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:32,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:32,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:32,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:32,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:32,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:32,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T02:56:33,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/04b78abb6d3f4c998449e24ecdad0ca8 2024-11-28T02:56:33,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/70153d7e446c42118fae97cf875093bc is 50, key is test_row_0/C:col10/1732762591816/Put/seqid=0 2024-11-28T02:56:33,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742422_1598 (size=12151) 2024-11-28T02:56:33,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-28T02:56:33,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:33,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:33,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:33,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:33,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:33,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-28T02:56:33,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:33,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:33,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:33,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-28T02:56:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:33,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:56:33,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/70153d7e446c42118fae97cf875093bc 2024-11-28T02:56:33,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/fae6c8cd9716487d88582b4ab9a42d56 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/fae6c8cd9716487d88582b4ab9a42d56 2024-11-28T02:56:33,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/fae6c8cd9716487d88582b4ab9a42d56, entries=200, sequenceid=206, filesize=38.8 K 2024-11-28T02:56:33,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/04b78abb6d3f4c998449e24ecdad0ca8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/04b78abb6d3f4c998449e24ecdad0ca8 2024-11-28T02:56:33,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/04b78abb6d3f4c998449e24ecdad0ca8, entries=150, sequenceid=206, filesize=11.9 K 2024-11-28T02:56:33,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/70153d7e446c42118fae97cf875093bc as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/70153d7e446c42118fae97cf875093bc 2024-11-28T02:56:33,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/70153d7e446c42118fae97cf875093bc, entries=150, sequenceid=206, filesize=11.9 K 2024-11-28T02:56:33,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 1645ms, sequenceid=206, compaction requested=false 2024-11-28T02:56:33,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T02:56:33,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-28T02:56:33,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:33,599 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T02:56:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:33,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112859f9a09e16ee4b8c98e44d8cc07a1128_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762591835/Put/seqid=0 2024-11-28T02:56:33,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742423_1599 (size=12304) 2024-11-28T02:56:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:33,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:34,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:34,015 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112859f9a09e16ee4b8c98e44d8cc07a1128_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112859f9a09e16ee4b8c98e44d8cc07a1128_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:34,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/8ccfe1043c4a4208a28689af91d86892, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:34,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/8ccfe1043c4a4208a28689af91d86892 is 175, key is test_row_0/A:col10/1732762591835/Put/seqid=0 2024-11-28T02:56:34,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762654013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742424_1600 (size=31105) 2024-11-28T02:56:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762654013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762654014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762654015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,021 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/8ccfe1043c4a4208a28689af91d86892 2024-11-28T02:56:34,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/314661b5fbea49eea5b798546b970c0d is 50, key is test_row_0/B:col10/1732762591835/Put/seqid=0 2024-11-28T02:56:34,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742425_1601 (size=12151) 2024-11-28T02:56:34,034 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/314661b5fbea49eea5b798546b970c0d 2024-11-28T02:56:34,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/ffdc5e877b444e04bf57c4793759f408 is 50, key is test_row_0/C:col10/1732762591835/Put/seqid=0 2024-11-28T02:56:34,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742426_1602 (size=12151) 2024-11-28T02:56:34,049 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/ffdc5e877b444e04bf57c4793759f408 2024-11-28T02:56:34,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/8ccfe1043c4a4208a28689af91d86892 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/8ccfe1043c4a4208a28689af91d86892 2024-11-28T02:56:34,056 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/8ccfe1043c4a4208a28689af91d86892, entries=150, sequenceid=226, filesize=30.4 K 2024-11-28T02:56:34,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/314661b5fbea49eea5b798546b970c0d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/314661b5fbea49eea5b798546b970c0d 2024-11-28T02:56:34,060 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/314661b5fbea49eea5b798546b970c0d, entries=150, sequenceid=226, filesize=11.9 K 2024-11-28T02:56:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/ffdc5e877b444e04bf57c4793759f408 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/ffdc5e877b444e04bf57c4793759f408 2024-11-28T02:56:34,064 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/ffdc5e877b444e04bf57c4793759f408, entries=150, sequenceid=226, filesize=11.9 K 2024-11-28T02:56:34,064 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 466ms, sequenceid=226, compaction requested=true 2024-11-28T02:56:34,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:34,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:34,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-28T02:56:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-28T02:56:34,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-28T02:56:34,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6870 sec 2024-11-28T02:56:34,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.6920 sec 2024-11-28T02:56:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:34,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T02:56:34,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:34,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:34,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:34,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:34,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:34,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:34,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112832980fef216a400b826bc47e915b953e_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762594011/Put/seqid=0 2024-11-28T02:56:34,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742427_1603 (size=14794) 2024-11-28T02:56:34,138 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:34,141 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112832980fef216a400b826bc47e915b953e_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112832980fef216a400b826bc47e915b953e_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:34,141 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/c03d4c9d76f948dfb7d0abb874a7b1cd, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:34,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/c03d4c9d76f948dfb7d0abb874a7b1cd is 175, key is test_row_0/A:col10/1732762594011/Put/seqid=0 2024-11-28T02:56:34,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742428_1604 (size=39749) 2024-11-28T02:56:34,146 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/c03d4c9d76f948dfb7d0abb874a7b1cd 2024-11-28T02:56:34,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/7dd110f9876e4e5eae08ec038e718170 is 50, key is test_row_0/B:col10/1732762594011/Put/seqid=0 2024-11-28T02:56:34,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762654146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762654147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742429_1605 (size=12151) 2024-11-28T02:56:34,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762654150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762654151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762654256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762654256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762654266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762654266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762654466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762654466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762654476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762654476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T02:56:34,526 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:34,526 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T02:56:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-28T02:56:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T02:56:34,527 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T02:56:34,528 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T02:56:34,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T02:56:34,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/7dd110f9876e4e5eae08ec038e718170 2024-11-28T02:56:34,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/b3d8be8a9877436bbb157785deb7d35f is 50, key is test_row_0/C:col10/1732762594011/Put/seqid=0 2024-11-28T02:56:34,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742430_1606 (size=12151) 2024-11-28T02:56:34,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/b3d8be8a9877436bbb157785deb7d35f 2024-11-28T02:56:34,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/c03d4c9d76f948dfb7d0abb874a7b1cd as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/c03d4c9d76f948dfb7d0abb874a7b1cd 2024-11-28T02:56:34,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/c03d4c9d76f948dfb7d0abb874a7b1cd, entries=200, sequenceid=242, filesize=38.8 K 2024-11-28T02:56:34,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/7dd110f9876e4e5eae08ec038e718170 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/7dd110f9876e4e5eae08ec038e718170 2024-11-28T02:56:34,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/7dd110f9876e4e5eae08ec038e718170, entries=150, sequenceid=242, filesize=11.9 K 2024-11-28T02:56:34,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/b3d8be8a9877436bbb157785deb7d35f as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b3d8be8a9877436bbb157785deb7d35f 2024-11-28T02:56:34,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b3d8be8a9877436bbb157785deb7d35f, entries=150, sequenceid=242, filesize=11.9 K 2024-11-28T02:56:34,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 445ms, sequenceid=242, compaction requested=true 2024-11-28T02:56:34,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:34,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:56:34,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:34,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:56:34,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:34,574 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:56:34,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:56:34,575 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:56:34,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:34,575 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:56:34,575 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/A is initiating minor compaction (all files) 2024-11-28T02:56:34,575 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/A in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:34,575 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/61cff6407bdb49dbbcbb678d19095d88, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/fae6c8cd9716487d88582b4ab9a42d56, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/8ccfe1043c4a4208a28689af91d86892, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/c03d4c9d76f948dfb7d0abb874a7b1cd] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=138.8 K 2024-11-28T02:56:34,576 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:34,576 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/61cff6407bdb49dbbcbb678d19095d88, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/fae6c8cd9716487d88582b4ab9a42d56, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/8ccfe1043c4a4208a28689af91d86892, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/c03d4c9d76f948dfb7d0abb874a7b1cd] 2024-11-28T02:56:34,576 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:56:34,576 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61cff6407bdb49dbbcbb678d19095d88, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732762589016 2024-11-28T02:56:34,576 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/B is initiating minor compaction (all files) 2024-11-28T02:56:34,576 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/B in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:34,576 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/9f293e870a99435da6e6be1f4372bdf5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/04b78abb6d3f4c998449e24ecdad0ca8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/314661b5fbea49eea5b798546b970c0d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/7dd110f9876e4e5eae08ec038e718170] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=47.9 K 2024-11-28T02:56:34,576 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting fae6c8cd9716487d88582b4ab9a42d56, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732762591184 2024-11-28T02:56:34,576 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ccfe1043c4a4208a28689af91d86892, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732762591835 2024-11-28T02:56:34,576 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f293e870a99435da6e6be1f4372bdf5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732762589016 2024-11-28T02:56:34,576 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c03d4c9d76f948dfb7d0abb874a7b1cd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732762594006 2024-11-28T02:56:34,576 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b78abb6d3f4c998449e24ecdad0ca8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732762591184 2024-11-28T02:56:34,577 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 314661b5fbea49eea5b798546b970c0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732762591835 2024-11-28T02:56:34,577 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dd110f9876e4e5eae08ec038e718170, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732762594006 2024-11-28T02:56:34,581 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:34,582 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#B#compaction#508 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:34,583 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/ea8de1fbc7cd49758f275a084a33e407 is 50, key is test_row_0/B:col10/1732762594011/Put/seqid=0 2024-11-28T02:56:34,584 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112819bb9a23ca734a7eb7dd830d8cffe332_3bd4185211d8fb1e40f0aca1aa4a4576 store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:34,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742431_1607 (size=12731) 2024-11-28T02:56:34,586 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112819bb9a23ca734a7eb7dd830d8cffe332_3bd4185211d8fb1e40f0aca1aa4a4576, store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:34,586 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112819bb9a23ca734a7eb7dd830d8cffe332_3bd4185211d8fb1e40f0aca1aa4a4576 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:34,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742432_1608 (size=4469) 2024-11-28T02:56:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T02:56:34,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-28T02:56:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:34,680 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T02:56:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:34,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b84570ee38af4c7681831360b0e15a5c_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762594144/Put/seqid=0 2024-11-28T02:56:34,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742433_1609 (size=12354) 2024-11-28T02:56:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:34,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. as already flushing 2024-11-28T02:56:34,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762654796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762654797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762654801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762654802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T02:56:34,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762654906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762654906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762654916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762654916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:34,989 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/ea8de1fbc7cd49758f275a084a33e407 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/ea8de1fbc7cd49758f275a084a33e407 2024-11-28T02:56:34,992 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/B of 3bd4185211d8fb1e40f0aca1aa4a4576 into ea8de1fbc7cd49758f275a084a33e407(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:34,992 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:34,992 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/B, priority=12, startTime=1732762594574; duration=0sec 2024-11-28T02:56:34,993 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:34,993 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:B 2024-11-28T02:56:34,993 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:56:34,993 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:56:34,993 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/C is initiating minor compaction (all files) 2024-11-28T02:56:34,993 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/C in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:34,994 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/72cd1ca01aa94bc39ff525f68bfd482e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/70153d7e446c42118fae97cf875093bc, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/ffdc5e877b444e04bf57c4793759f408, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b3d8be8a9877436bbb157785deb7d35f] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=47.9 K 2024-11-28T02:56:34,994 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 72cd1ca01aa94bc39ff525f68bfd482e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732762589016 2024-11-28T02:56:34,994 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 70153d7e446c42118fae97cf875093bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732762591184 2024-11-28T02:56:34,994 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdc5e877b444e04bf57c4793759f408, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732762591835 2024-11-28T02:56:34,995 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#A#compaction#507 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:34,995 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting b3d8be8a9877436bbb157785deb7d35f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732762594006 2024-11-28T02:56:34,995 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/30bd02f7ec3a4dcc88b5a9d6e62de8c3 is 175, key is test_row_0/A:col10/1732762594011/Put/seqid=0 2024-11-28T02:56:34,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742434_1610 (size=31685) 2024-11-28T02:56:35,002 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#C#compaction#510 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:35,002 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/b4f109c9d80946cab7ccc8fa534fdc7d is 50, key is test_row_0/C:col10/1732762594011/Put/seqid=0 2024-11-28T02:56:35,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742435_1611 (size=12731) 2024-11-28T02:56:35,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:35,091 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b84570ee38af4c7681831360b0e15a5c_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b84570ee38af4c7681831360b0e15a5c_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:35,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/58e5bd7127074038a80c82df724ed0d4, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:35,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/58e5bd7127074038a80c82df724ed0d4 is 175, key is test_row_0/A:col10/1732762594144/Put/seqid=0 2024-11-28T02:56:35,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742436_1612 (size=31155) 2024-11-28T02:56:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762655116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762655116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:35,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762655126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:35,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762655126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T02:56:35,402 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/30bd02f7ec3a4dcc88b5a9d6e62de8c3 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/30bd02f7ec3a4dcc88b5a9d6e62de8c3 2024-11-28T02:56:35,405 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/A of 3bd4185211d8fb1e40f0aca1aa4a4576 into 30bd02f7ec3a4dcc88b5a9d6e62de8c3(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:35,405 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:35,405 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/A, priority=12, startTime=1732762594574; duration=0sec 2024-11-28T02:56:35,406 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:35,406 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:A 2024-11-28T02:56:35,408 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/b4f109c9d80946cab7ccc8fa534fdc7d as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b4f109c9d80946cab7ccc8fa534fdc7d 2024-11-28T02:56:35,411 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/C of 3bd4185211d8fb1e40f0aca1aa4a4576 into b4f109c9d80946cab7ccc8fa534fdc7d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:35,411 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:35,411 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/C, priority=12, startTime=1732762594574; duration=0sec 2024-11-28T02:56:35,411 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:35,411 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:C 2024-11-28T02:56:35,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762655426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762655426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:35,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762655436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:35,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762655436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:35,495 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/58e5bd7127074038a80c82df724ed0d4 2024-11-28T02:56:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/3973aed48e6c46ba8507fc2c4afe2187 is 50, key is test_row_0/B:col10/1732762594144/Put/seqid=0 2024-11-28T02:56:35,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742437_1613 (size=12201) 2024-11-28T02:56:35,582 INFO [Thread-2415 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2415. 2024-11-28T02:56:35,582 DEBUG [Thread-2415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:35,582 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:35,582 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:35,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:35,583 INFO [Thread-2417 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2417. 2024-11-28T02:56:35,583 DEBUG [Thread-2417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:35,583 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:35,583 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:35,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:35,584 INFO [Thread-2423 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2423. 2024-11-28T02:56:35,584 DEBUG [Thread-2423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:35,584 DEBUG [Thread-2423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:35,584 DEBUG [Thread-2423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:35,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:35,585 INFO [Thread-2421 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2421. 2024-11-28T02:56:35,585 DEBUG [Thread-2421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:35,586 DEBUG [Thread-2421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:35,586 DEBUG [Thread-2421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:35,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:35,587 INFO [Thread-2419 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2419. 2024-11-28T02:56:35,587 DEBUG [Thread-2419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:35,587 DEBUG [Thread-2419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:35,587 DEBUG [Thread-2419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:35,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T02:56:35,906 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/3973aed48e6c46ba8507fc2c4afe2187 2024-11-28T02:56:35,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/60ec60cc45eb4c01bba09e07cfbde07c is 50, key is test_row_0/C:col10/1732762594144/Put/seqid=0 2024-11-28T02:56:35,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742438_1614 (size=12201) 2024-11-28T02:56:35,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:35,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52280 deadline: 1732762655936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52240 deadline: 1732762655936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:35,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:35,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:35,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52268 deadline: 1732762655946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:35,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52246 deadline: 1732762655946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:35,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:36,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T02:56:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52204 deadline: 1732762656097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 2024-11-28T02:56:36,099 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 978bd30859c3,41631,1732762419946 for 'test_row_2' in 3bd4185211d8fb1e40f0aca1aa4a4576 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T02:56:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 , the old value is region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:56:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., hostname=978bd30859c3,41631,1732762419946, seqNum=5 because the exception is null or not the one we care about 2024-11-28T02:56:36,324 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/60ec60cc45eb4c01bba09e07cfbde07c 2024-11-28T02:56:36,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/58e5bd7127074038a80c82df724ed0d4 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/58e5bd7127074038a80c82df724ed0d4 2024-11-28T02:56:36,340 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/58e5bd7127074038a80c82df724ed0d4, entries=150, sequenceid=258, filesize=30.4 K 2024-11-28T02:56:36,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/3973aed48e6c46ba8507fc2c4afe2187 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/3973aed48e6c46ba8507fc2c4afe2187 2024-11-28T02:56:36,345 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/3973aed48e6c46ba8507fc2c4afe2187, entries=150, sequenceid=258, filesize=11.9 K 2024-11-28T02:56:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/60ec60cc45eb4c01bba09e07cfbde07c as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/60ec60cc45eb4c01bba09e07cfbde07c 2024-11-28T02:56:36,350 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/60ec60cc45eb4c01bba09e07cfbde07c, entries=150, sequenceid=258, filesize=11.9 K 2024-11-28T02:56:36,351 INFO [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 1670ms, sequenceid=258, compaction requested=false 2024-11-28T02:56:36,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:36,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:36,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/978bd30859c3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-28T02:56:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-28T02:56:36,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-28T02:56:36,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8240 sec 2024-11-28T02:56:36,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.8290 sec 2024-11-28T02:56:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T02:56:36,678 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:36,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] regionserver.HRegion(8855): Flush requested on 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:36,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T02:56:36,949 INFO [Thread-2410 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2410. 2024-11-28T02:56:36,949 DEBUG [Thread-2410 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:36,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:36,949 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:36,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:36,950 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:36,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:36,958 INFO [Thread-2406 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2406. 2024-11-28T02:56:36,958 INFO [Thread-2408 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2408. 2024-11-28T02:56:36,958 DEBUG [Thread-2406 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:36,958 DEBUG [Thread-2408 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:36,958 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:36,958 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:36,958 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:36,958 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:36,958 INFO [Thread-2412 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2412. 2024-11-28T02:56:36,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:36,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:36,958 DEBUG [Thread-2412 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:36,958 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:36,958 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:36,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:36,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128cd9c1bc6a004421ebae61d58e4f95ff0_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762594801/Put/seqid=0 2024-11-28T02:56:36,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742439_1615 (size=12454) 2024-11-28T02:56:37,364 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:37,373 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128cd9c1bc6a004421ebae61d58e4f95ff0_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128cd9c1bc6a004421ebae61d58e4f95ff0_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:37,375 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/6870542e0b05407b9bf9935909959699, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:37,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/6870542e0b05407b9bf9935909959699 is 175, key is test_row_0/A:col10/1732762594801/Put/seqid=0 2024-11-28T02:56:37,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742440_1616 (size=31255) 2024-11-28T02:56:37,781 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/6870542e0b05407b9bf9935909959699 2024-11-28T02:56:37,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/2d51130a5c674d83885df1d23b91d7fb is 50, key is test_row_0/B:col10/1732762594801/Put/seqid=0 2024-11-28T02:56:37,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742441_1617 (size=12301) 2024-11-28T02:56:37,998 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T02:56:38,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/2d51130a5c674d83885df1d23b91d7fb 2024-11-28T02:56:38,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/8e037ce7fc6d48c59544e62e06f745e5 is 50, key is test_row_0/C:col10/1732762594801/Put/seqid=0 2024-11-28T02:56:38,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742442_1618 (size=12301) 2024-11-28T02:56:38,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/8e037ce7fc6d48c59544e62e06f745e5 2024-11-28T02:56:38,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/6870542e0b05407b9bf9935909959699 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/6870542e0b05407b9bf9935909959699 2024-11-28T02:56:38,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/6870542e0b05407b9bf9935909959699, entries=150, sequenceid=278, filesize=30.5 K 2024-11-28T02:56:38,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/2d51130a5c674d83885df1d23b91d7fb as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/2d51130a5c674d83885df1d23b91d7fb 2024-11-28T02:56:38,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/2d51130a5c674d83885df1d23b91d7fb, entries=150, sequenceid=278, filesize=12.0 K 2024-11-28T02:56:38,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/8e037ce7fc6d48c59544e62e06f745e5 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/8e037ce7fc6d48c59544e62e06f745e5 2024-11-28T02:56:38,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/8e037ce7fc6d48c59544e62e06f745e5, entries=150, sequenceid=278, filesize=12.0 K 2024-11-28T02:56:38,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 1688ms, sequenceid=278, compaction requested=true 2024-11-28T02:56:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:56:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:38,637 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:56:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3bd4185211d8fb1e40f0aca1aa4a4576:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T02:56:38,637 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:38,638 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:38,638 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:38,638 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/A is initiating minor compaction (all files) 2024-11-28T02:56:38,638 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/B is initiating minor compaction (all files) 2024-11-28T02:56:38,638 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/A in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:38,638 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/B in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:38,638 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/30bd02f7ec3a4dcc88b5a9d6e62de8c3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/58e5bd7127074038a80c82df724ed0d4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/6870542e0b05407b9bf9935909959699] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=91.9 K 2024-11-28T02:56:38,638 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/ea8de1fbc7cd49758f275a084a33e407, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/3973aed48e6c46ba8507fc2c4afe2187, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/2d51130a5c674d83885df1d23b91d7fb] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=36.4 K 2024-11-28T02:56:38,638 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:38,638 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. files: [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/30bd02f7ec3a4dcc88b5a9d6e62de8c3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/58e5bd7127074038a80c82df724ed0d4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/6870542e0b05407b9bf9935909959699] 2024-11-28T02:56:38,639 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting ea8de1fbc7cd49758f275a084a33e407, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732762594006 2024-11-28T02:56:38,639 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30bd02f7ec3a4dcc88b5a9d6e62de8c3, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732762594006 2024-11-28T02:56:38,639 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58e5bd7127074038a80c82df724ed0d4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732762594144 2024-11-28T02:56:38,639 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3973aed48e6c46ba8507fc2c4afe2187, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732762594144 2024-11-28T02:56:38,639 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6870542e0b05407b9bf9935909959699, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732762594793 2024-11-28T02:56:38,639 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d51130a5c674d83885df1d23b91d7fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732762594793 2024-11-28T02:56:38,647 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#B#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:38,647 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:38,647 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/0644a85962794e2586424f4a5b2f0e92 is 50, key is test_row_0/B:col10/1732762594801/Put/seqid=0 2024-11-28T02:56:38,651 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112893a506edd89440278cfa2974921bfbd7_3bd4185211d8fb1e40f0aca1aa4a4576 store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:38,658 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112893a506edd89440278cfa2974921bfbd7_3bd4185211d8fb1e40f0aca1aa4a4576, store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:38,658 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112893a506edd89440278cfa2974921bfbd7_3bd4185211d8fb1e40f0aca1aa4a4576 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:38,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742443_1619 (size=12983) 2024-11-28T02:56:38,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742444_1620 (size=4469) 2024-11-28T02:56:39,069 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#A#compaction#517 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:39,070 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/51e144ee7ed24e4f9748ecd8fb617957 is 175, key is test_row_0/A:col10/1732762594801/Put/seqid=0 2024-11-28T02:56:39,071 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/0644a85962794e2586424f4a5b2f0e92 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/0644a85962794e2586424f4a5b2f0e92 2024-11-28T02:56:39,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742445_1621 (size=31937) 2024-11-28T02:56:39,076 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/B of 3bd4185211d8fb1e40f0aca1aa4a4576 into 0644a85962794e2586424f4a5b2f0e92(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:39,076 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:39,076 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/B, priority=13, startTime=1732762598637; duration=0sec 2024-11-28T02:56:39,076 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:56:39,076 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:B 2024-11-28T02:56:39,076 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:56:39,077 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:56:39,077 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1541): 3bd4185211d8fb1e40f0aca1aa4a4576/C is initiating minor compaction (all files) 2024-11-28T02:56:39,077 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3bd4185211d8fb1e40f0aca1aa4a4576/C in TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:39,077 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b4f109c9d80946cab7ccc8fa534fdc7d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/60ec60cc45eb4c01bba09e07cfbde07c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/8e037ce7fc6d48c59544e62e06f745e5] into tmpdir=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp, totalSize=36.4 K 2024-11-28T02:56:39,078 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting b4f109c9d80946cab7ccc8fa534fdc7d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732762594006 2024-11-28T02:56:39,078 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ec60cc45eb4c01bba09e07cfbde07c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732762594144 2024-11-28T02:56:39,078 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e037ce7fc6d48c59544e62e06f745e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732762594793 2024-11-28T02:56:39,085 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3bd4185211d8fb1e40f0aca1aa4a4576#C#compaction#518 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:56:39,085 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/4719b1a1ba59480dafc9f6eca58aa4b8 is 50, key is test_row_0/C:col10/1732762594801/Put/seqid=0 2024-11-28T02:56:39,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742446_1622 (size=12983) 2024-11-28T02:56:39,480 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/51e144ee7ed24e4f9748ecd8fb617957 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/51e144ee7ed24e4f9748ecd8fb617957 2024-11-28T02:56:39,487 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/A of 3bd4185211d8fb1e40f0aca1aa4a4576 into 51e144ee7ed24e4f9748ecd8fb617957(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:39,487 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:39,487 INFO [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/A, priority=13, startTime=1732762598637; duration=0sec 2024-11-28T02:56:39,487 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:39,487 DEBUG [RS:0;978bd30859c3:41631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:A 2024-11-28T02:56:39,494 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/4719b1a1ba59480dafc9f6eca58aa4b8 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/4719b1a1ba59480dafc9f6eca58aa4b8 2024-11-28T02:56:39,499 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3bd4185211d8fb1e40f0aca1aa4a4576/C of 3bd4185211d8fb1e40f0aca1aa4a4576 into 4719b1a1ba59480dafc9f6eca58aa4b8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:56:39,499 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3bd4185211d8fb1e40f0aca1aa4a4576: 2024-11-28T02:56:39,499 INFO [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., storeName=3bd4185211d8fb1e40f0aca1aa4a4576/C, priority=13, startTime=1732762598637; duration=0sec 2024-11-28T02:56:39,499 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:56:39,499 DEBUG [RS:0;978bd30859c3:41631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3bd4185211d8fb1e40f0aca1aa4a4576:C 2024-11-28T02:56:46,121 INFO [Thread-2404 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2404. 2024-11-28T02:56:46,121 DEBUG [Thread-2404 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T02:56:46,121 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:46,121 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:46,121 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:46,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T02:56:46,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-28T02:56:46,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-28T02:56:46,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-28T02:56:46,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-28T02:56:46,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-28T02:56:46,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T02:56:46,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3431 2024-11-28T02:56:46,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3461 2024-11-28T02:56:46,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3369 2024-11-28T02:56:46,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3417 2024-11-28T02:56:46,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3448 2024-11-28T02:56:46,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T02:56:46,123 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:56:46,123 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:56:46,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:46,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:46,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:46,125 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T02:56:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T02:56:46,179 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762606179"}]},"ts":"1732762606179"} 2024-11-28T02:56:46,183 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T02:56:46,183 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T02:56:46,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T02:56:46,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, UNASSIGN}] 2024-11-28T02:56:46,185 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, UNASSIGN 2024-11-28T02:56:46,186 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=3bd4185211d8fb1e40f0aca1aa4a4576, regionState=CLOSING, regionLocation=978bd30859c3,41631,1732762419946 2024-11-28T02:56:46,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, UNASSIGN because future has completed 2024-11-28T02:56:46,188 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T02:56:46,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946}] 2024-11-28T02:56:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T02:56:46,340 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:46,340 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T02:56:46,340 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing 3bd4185211d8fb1e40f0aca1aa4a4576, disabling compactions & flushes 2024-11-28T02:56:46,340 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:46,340 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:46,340 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. after waiting 0 ms 2024-11-28T02:56:46,340 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:46,340 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing 3bd4185211d8fb1e40f0aca1aa4a4576 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-28T02:56:46,341 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=A 2024-11-28T02:56:46,341 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:46,341 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=B 2024-11-28T02:56:46,341 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:46,341 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3bd4185211d8fb1e40f0aca1aa4a4576, store=C 2024-11-28T02:56:46,341 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T02:56:46,348 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128dd9270f6be3f4625988855483dc8d585_3bd4185211d8fb1e40f0aca1aa4a4576 is 50, key is test_row_0/A:col10/1732762596956/Put/seqid=0 2024-11-28T02:56:46,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742447_1623 (size=12454) 2024-11-28T02:56:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T02:56:46,753 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:56:46,756 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128dd9270f6be3f4625988855483dc8d585_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dd9270f6be3f4625988855483dc8d585_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:46,757 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/bdd425a63f4242e3a23172e4f8ae35fa, store: [table=TestAcidGuarantees family=A region=3bd4185211d8fb1e40f0aca1aa4a4576] 2024-11-28T02:56:46,757 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/bdd425a63f4242e3a23172e4f8ae35fa is 175, key is test_row_0/A:col10/1732762596956/Put/seqid=0 2024-11-28T02:56:46,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742448_1624 (size=31255) 2024-11-28T02:56:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T02:56:47,161 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/bdd425a63f4242e3a23172e4f8ae35fa 2024-11-28T02:56:47,169 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/a7c20467f58145fa88ddcbec39c63fbc is 50, key is test_row_0/B:col10/1732762596956/Put/seqid=0 2024-11-28T02:56:47,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742449_1625 (size=12301) 2024-11-28T02:56:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T02:56:47,574 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/a7c20467f58145fa88ddcbec39c63fbc 2024-11-28T02:56:47,579 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/d77b756eddbb4f3489bccfd8f8cd4d79 is 50, key is test_row_0/C:col10/1732762596956/Put/seqid=0 2024-11-28T02:56:47,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742450_1626 (size=12301) 2024-11-28T02:56:47,982 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/d77b756eddbb4f3489bccfd8f8cd4d79 2024-11-28T02:56:47,985 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/A/bdd425a63f4242e3a23172e4f8ae35fa as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/bdd425a63f4242e3a23172e4f8ae35fa 2024-11-28T02:56:47,988 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/bdd425a63f4242e3a23172e4f8ae35fa, entries=150, sequenceid=288, filesize=30.5 K 2024-11-28T02:56:47,989 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/B/a7c20467f58145fa88ddcbec39c63fbc as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/a7c20467f58145fa88ddcbec39c63fbc 2024-11-28T02:56:47,992 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/a7c20467f58145fa88ddcbec39c63fbc, entries=150, sequenceid=288, filesize=12.0 K 2024-11-28T02:56:47,993 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/.tmp/C/d77b756eddbb4f3489bccfd8f8cd4d79 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/d77b756eddbb4f3489bccfd8f8cd4d79 2024-11-28T02:56:47,995 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/d77b756eddbb4f3489bccfd8f8cd4d79, entries=150, sequenceid=288, filesize=12.0 K 2024-11-28T02:56:47,996 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 1656ms, sequenceid=288, compaction requested=false 2024-11-28T02:56:47,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/51b6ebc3609b48f2813b1860b21f7617, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/cfb8373de3464bbca86fbc26f67c99e3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/2a2e10fe641e41f18a6110984eadc552, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/56832ce88bef411ea7835026f7ef7db1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/5a8cf96d561a4f14b7e9da2f7e66e26a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/478fea258e824c138b7c55e28d26bbb0, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/f33fe37c40004089907622ebaa552de7, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/d8bd689d405049868e2070293622f872, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/f9e803423f3b432d892ba6e0f8b4f79a, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/075fd2776304406baa2edad4494f00cf, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/2b70bf3b241b40bdaee4e05ad97238ad, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/11b5b60ecaca45ec92f6948922fd89af, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/08b1465d53b142a7b3483bc5e3ab0d88, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/61cff6407bdb49dbbcbb678d19095d88, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/fae6c8cd9716487d88582b4ab9a42d56, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/8ccfe1043c4a4208a28689af91d86892, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/c03d4c9d76f948dfb7d0abb874a7b1cd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/30bd02f7ec3a4dcc88b5a9d6e62de8c3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/58e5bd7127074038a80c82df724ed0d4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/6870542e0b05407b9bf9935909959699] to archive 2024-11-28T02:56:47,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:56:47,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/51b6ebc3609b48f2813b1860b21f7617 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/51b6ebc3609b48f2813b1860b21f7617 2024-11-28T02:56:47,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/cfb8373de3464bbca86fbc26f67c99e3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/cfb8373de3464bbca86fbc26f67c99e3 2024-11-28T02:56:48,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/2a2e10fe641e41f18a6110984eadc552 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/2a2e10fe641e41f18a6110984eadc552 2024-11-28T02:56:48,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/56832ce88bef411ea7835026f7ef7db1 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/56832ce88bef411ea7835026f7ef7db1 2024-11-28T02:56:48,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/5a8cf96d561a4f14b7e9da2f7e66e26a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/5a8cf96d561a4f14b7e9da2f7e66e26a 2024-11-28T02:56:48,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/478fea258e824c138b7c55e28d26bbb0 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/478fea258e824c138b7c55e28d26bbb0 2024-11-28T02:56:48,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/f33fe37c40004089907622ebaa552de7 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/f33fe37c40004089907622ebaa552de7 2024-11-28T02:56:48,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/d8bd689d405049868e2070293622f872 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/d8bd689d405049868e2070293622f872 2024-11-28T02:56:48,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/f9e803423f3b432d892ba6e0f8b4f79a to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/f9e803423f3b432d892ba6e0f8b4f79a 2024-11-28T02:56:48,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/075fd2776304406baa2edad4494f00cf to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/075fd2776304406baa2edad4494f00cf 2024-11-28T02:56:48,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/2b70bf3b241b40bdaee4e05ad97238ad to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/2b70bf3b241b40bdaee4e05ad97238ad 2024-11-28T02:56:48,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/11b5b60ecaca45ec92f6948922fd89af to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/11b5b60ecaca45ec92f6948922fd89af 2024-11-28T02:56:48,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/08b1465d53b142a7b3483bc5e3ab0d88 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/08b1465d53b142a7b3483bc5e3ab0d88 2024-11-28T02:56:48,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/61cff6407bdb49dbbcbb678d19095d88 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/61cff6407bdb49dbbcbb678d19095d88 2024-11-28T02:56:48,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/fae6c8cd9716487d88582b4ab9a42d56 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/fae6c8cd9716487d88582b4ab9a42d56 2024-11-28T02:56:48,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/8ccfe1043c4a4208a28689af91d86892 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/8ccfe1043c4a4208a28689af91d86892 2024-11-28T02:56:48,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/c03d4c9d76f948dfb7d0abb874a7b1cd to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/c03d4c9d76f948dfb7d0abb874a7b1cd 2024-11-28T02:56:48,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/30bd02f7ec3a4dcc88b5a9d6e62de8c3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/30bd02f7ec3a4dcc88b5a9d6e62de8c3 2024-11-28T02:56:48,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/58e5bd7127074038a80c82df724ed0d4 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/58e5bd7127074038a80c82df724ed0d4 2024-11-28T02:56:48,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/6870542e0b05407b9bf9935909959699 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/6870542e0b05407b9bf9935909959699 2024-11-28T02:56:48,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/84908fb9201b4520bf45651ff1b4e79f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/08949dd3406c4ea7b374a47b2f25d4d4, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/1a5ce2df591141c0828b93c600414492, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/ada3e02495544f8cb67981470c12eb7d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/4f8cb9438503414db74c17e3ec35adbd, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/5b558bd92d2742a19d73b50b29c737df, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/adc91eed5056408893ba40a6746dc9d1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/0824323d4e75497690155acb794cd472, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/24fc2e5102fc49849aebd90c59ed7a58, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/f15fec348c1249fa8a5a054a03b6da93, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/e150c74768ac42e39d08d524e8565cf1, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/d2abd584604f44e7b79a6544d7b3e632, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/9f293e870a99435da6e6be1f4372bdf5, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/48f67db3dbd944ddb9d975cc3d158e92, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/04b78abb6d3f4c998449e24ecdad0ca8, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/314661b5fbea49eea5b798546b970c0d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/ea8de1fbc7cd49758f275a084a33e407, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/7dd110f9876e4e5eae08ec038e718170, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/3973aed48e6c46ba8507fc2c4afe2187, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/2d51130a5c674d83885df1d23b91d7fb] to archive 2024-11-28T02:56:48,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:56:48,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/84908fb9201b4520bf45651ff1b4e79f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/84908fb9201b4520bf45651ff1b4e79f 2024-11-28T02:56:48,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/08949dd3406c4ea7b374a47b2f25d4d4 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/08949dd3406c4ea7b374a47b2f25d4d4 2024-11-28T02:56:48,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/1a5ce2df591141c0828b93c600414492 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/1a5ce2df591141c0828b93c600414492 2024-11-28T02:56:48,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/ada3e02495544f8cb67981470c12eb7d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/ada3e02495544f8cb67981470c12eb7d 2024-11-28T02:56:48,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/4f8cb9438503414db74c17e3ec35adbd to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/4f8cb9438503414db74c17e3ec35adbd 2024-11-28T02:56:48,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/5b558bd92d2742a19d73b50b29c737df to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/5b558bd92d2742a19d73b50b29c737df 2024-11-28T02:56:48,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/adc91eed5056408893ba40a6746dc9d1 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/adc91eed5056408893ba40a6746dc9d1 2024-11-28T02:56:48,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/0824323d4e75497690155acb794cd472 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/0824323d4e75497690155acb794cd472 2024-11-28T02:56:48,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/24fc2e5102fc49849aebd90c59ed7a58 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/24fc2e5102fc49849aebd90c59ed7a58 2024-11-28T02:56:48,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/f15fec348c1249fa8a5a054a03b6da93 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/f15fec348c1249fa8a5a054a03b6da93 2024-11-28T02:56:48,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/e150c74768ac42e39d08d524e8565cf1 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/e150c74768ac42e39d08d524e8565cf1 2024-11-28T02:56:48,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/d2abd584604f44e7b79a6544d7b3e632 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/d2abd584604f44e7b79a6544d7b3e632 2024-11-28T02:56:48,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/9f293e870a99435da6e6be1f4372bdf5 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/9f293e870a99435da6e6be1f4372bdf5 2024-11-28T02:56:48,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/48f67db3dbd944ddb9d975cc3d158e92 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/48f67db3dbd944ddb9d975cc3d158e92 2024-11-28T02:56:48,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/04b78abb6d3f4c998449e24ecdad0ca8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/04b78abb6d3f4c998449e24ecdad0ca8 2024-11-28T02:56:48,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/314661b5fbea49eea5b798546b970c0d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/314661b5fbea49eea5b798546b970c0d 2024-11-28T02:56:48,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/ea8de1fbc7cd49758f275a084a33e407 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/ea8de1fbc7cd49758f275a084a33e407 2024-11-28T02:56:48,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/7dd110f9876e4e5eae08ec038e718170 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/7dd110f9876e4e5eae08ec038e718170 2024-11-28T02:56:48,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/3973aed48e6c46ba8507fc2c4afe2187 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/3973aed48e6c46ba8507fc2c4afe2187 2024-11-28T02:56:48,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/2d51130a5c674d83885df1d23b91d7fb to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/2d51130a5c674d83885df1d23b91d7fb 2024-11-28T02:56:48,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/888c8cd5dffa4aee8f28667a380d4eab, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/9392a9f97a9743188d5a36a65168f383, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/d580d0b289b54f2191faee71720267c3, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b7af8ec4a4c9480dab506ba43e41f718, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/9abec83a915d493097de3aec9f3d482f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/bf87fbba44304f91a2cb8933d53fb507, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/358455a6c21b4baca3dee56a137cc4cf, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/0502d658d4124b0489df736ea870a162, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/ba95b50621cf42caae23638a2090f638, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/84e864c6c32b4c1c945e201bb5905b0d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/5760b684e0374118a22fea48b1b9fb94, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/812ecd6d071e406d89f8232199ae331f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/72cd1ca01aa94bc39ff525f68bfd482e, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/01b63a37765a481d8d98061f62507ad2, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/70153d7e446c42118fae97cf875093bc, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/ffdc5e877b444e04bf57c4793759f408, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b4f109c9d80946cab7ccc8fa534fdc7d, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b3d8be8a9877436bbb157785deb7d35f, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/60ec60cc45eb4c01bba09e07cfbde07c, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/8e037ce7fc6d48c59544e62e06f745e5] to archive 2024-11-28T02:56:48,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:56:48,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/888c8cd5dffa4aee8f28667a380d4eab to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/888c8cd5dffa4aee8f28667a380d4eab 2024-11-28T02:56:48,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/9392a9f97a9743188d5a36a65168f383 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/9392a9f97a9743188d5a36a65168f383 2024-11-28T02:56:48,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/d580d0b289b54f2191faee71720267c3 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/d580d0b289b54f2191faee71720267c3 2024-11-28T02:56:48,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b7af8ec4a4c9480dab506ba43e41f718 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b7af8ec4a4c9480dab506ba43e41f718 2024-11-28T02:56:48,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/9abec83a915d493097de3aec9f3d482f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/9abec83a915d493097de3aec9f3d482f 2024-11-28T02:56:48,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/bf87fbba44304f91a2cb8933d53fb507 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/bf87fbba44304f91a2cb8933d53fb507 2024-11-28T02:56:48,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/358455a6c21b4baca3dee56a137cc4cf to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/358455a6c21b4baca3dee56a137cc4cf 2024-11-28T02:56:48,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/0502d658d4124b0489df736ea870a162 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/0502d658d4124b0489df736ea870a162 2024-11-28T02:56:48,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/ba95b50621cf42caae23638a2090f638 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/ba95b50621cf42caae23638a2090f638 2024-11-28T02:56:48,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/84e864c6c32b4c1c945e201bb5905b0d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/84e864c6c32b4c1c945e201bb5905b0d 2024-11-28T02:56:48,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/5760b684e0374118a22fea48b1b9fb94 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/5760b684e0374118a22fea48b1b9fb94 2024-11-28T02:56:48,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/812ecd6d071e406d89f8232199ae331f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/812ecd6d071e406d89f8232199ae331f 2024-11-28T02:56:48,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/72cd1ca01aa94bc39ff525f68bfd482e to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/72cd1ca01aa94bc39ff525f68bfd482e 2024-11-28T02:56:48,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/01b63a37765a481d8d98061f62507ad2 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/01b63a37765a481d8d98061f62507ad2 2024-11-28T02:56:48,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/70153d7e446c42118fae97cf875093bc to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/70153d7e446c42118fae97cf875093bc 2024-11-28T02:56:48,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/ffdc5e877b444e04bf57c4793759f408 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/ffdc5e877b444e04bf57c4793759f408 2024-11-28T02:56:48,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b4f109c9d80946cab7ccc8fa534fdc7d to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b4f109c9d80946cab7ccc8fa534fdc7d 2024-11-28T02:56:48,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b3d8be8a9877436bbb157785deb7d35f to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/b3d8be8a9877436bbb157785deb7d35f 2024-11-28T02:56:48,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/60ec60cc45eb4c01bba09e07cfbde07c to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/60ec60cc45eb4c01bba09e07cfbde07c 2024-11-28T02:56:48,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/8e037ce7fc6d48c59544e62e06f745e5 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/8e037ce7fc6d48c59544e62e06f745e5 2024-11-28T02:56:48,053 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/recovered.edits/291.seqid, newMaxSeqId=291, maxSeqId=4 2024-11-28T02:56:48,053 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. 2024-11-28T02:56:48,053 DEBUG [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for 3bd4185211d8fb1e40f0aca1aa4a4576: Waiting for close lock at 1732762606340Running coprocessor pre-close hooks at 1732762606340Disabling compacts and flushes for region at 1732762606340Disabling writes for close at 1732762606340Obtaining lock to block concurrent updates at 1732762606340Preparing flush snapshotting stores in 3bd4185211d8fb1e40f0aca1aa4a4576 at 1732762606340Finished memstore snapshotting TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732762606341 (+1 ms)Flushing stores of TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576. at 1732762606341Flushing 3bd4185211d8fb1e40f0aca1aa4a4576/A: creating writer at 1732762606341Flushing mob file 3bd4185211d8fb1e40f0aca1aa4a4576/A: appending metadata at 1732762606348 (+7 ms)Flushing mob file 3bd4185211d8fb1e40f0aca1aa4a4576/A: closing flushed file at 1732762606348Flushing 3bd4185211d8fb1e40f0aca1aa4a4576/A: appending metadata at 1732762606757 (+409 ms)Flushing 3bd4185211d8fb1e40f0aca1aa4a4576/A: closing flushed file at 1732762606757Flushing 3bd4185211d8fb1e40f0aca1aa4a4576/B: creating writer at 1732762607165 (+408 ms)Flushing 3bd4185211d8fb1e40f0aca1aa4a4576/B: appending metadata at 1732762607168 (+3 ms)Flushing 3bd4185211d8fb1e40f0aca1aa4a4576/B: closing flushed file at 1732762607168Flushing 3bd4185211d8fb1e40f0aca1aa4a4576/C: creating writer at 1732762607576 (+408 ms)Flushing 3bd4185211d8fb1e40f0aca1aa4a4576/C: appending metadata at 1732762607578 (+2 ms)Flushing 3bd4185211d8fb1e40f0aca1aa4a4576/C: closing flushed file at 1732762607578Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@544b6950: reopening flushed file at 1732762607985 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77a11475: reopening flushed file at 1732762607989 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e9dae55: reopening flushed file at 1732762607992 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 3bd4185211d8fb1e40f0aca1aa4a4576 in 1656ms, sequenceid=288, compaction requested=false at 1732762607996 (+4 ms)Writing region close event to WAL at 1732762608050 (+54 ms)Running coprocessor post-close hooks at 1732762608053 (+3 ms)Closed at 1732762608053 2024-11-28T02:56:48,055 INFO [RS_CLOSE_REGION-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed 3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,056 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=3bd4185211d8fb1e40f0aca1aa4a4576, regionState=CLOSED 2024-11-28T02:56:48,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 because future has completed 2024-11-28T02:56:48,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-28T02:56:48,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure 3bd4185211d8fb1e40f0aca1aa4a4576, server=978bd30859c3,41631,1732762419946 in 1.8690 sec 2024-11-28T02:56:48,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-11-28T02:56:48,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3bd4185211d8fb1e40f0aca1aa4a4576, UNASSIGN in 1.8740 sec 2024-11-28T02:56:48,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-28T02:56:48,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8770 sec 2024-11-28T02:56:48,062 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732762608062"}]},"ts":"1732762608062"} 2024-11-28T02:56:48,064 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T02:56:48,064 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T02:56:48,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9390 sec 2024-11-28T02:56:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T02:56:48,326 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:48,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T02:56:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:48,329 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T02:56:48,330 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:48,332 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,334 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C, FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/recovered.edits] 2024-11-28T02:56:48,336 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/51e144ee7ed24e4f9748ecd8fb617957 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/51e144ee7ed24e4f9748ecd8fb617957 2024-11-28T02:56:48,337 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/bdd425a63f4242e3a23172e4f8ae35fa to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/A/bdd425a63f4242e3a23172e4f8ae35fa 2024-11-28T02:56:48,338 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/0644a85962794e2586424f4a5b2f0e92 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/0644a85962794e2586424f4a5b2f0e92 2024-11-28T02:56:48,340 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/a7c20467f58145fa88ddcbec39c63fbc to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/B/a7c20467f58145fa88ddcbec39c63fbc 2024-11-28T02:56:48,341 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/4719b1a1ba59480dafc9f6eca58aa4b8 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/4719b1a1ba59480dafc9f6eca58aa4b8 2024-11-28T02:56:48,342 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/d77b756eddbb4f3489bccfd8f8cd4d79 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/C/d77b756eddbb4f3489bccfd8f8cd4d79 2024-11-28T02:56:48,344 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/recovered.edits/291.seqid to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576/recovered.edits/291.seqid 2024-11-28T02:56:48,345 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/default/TestAcidGuarantees/3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,345 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T02:56:48,345 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T02:56:48,346 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T02:56:48,348 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128252714d48dff414380d3494b533e758d_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128252714d48dff414380d3494b533e758d_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,349 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128281b7f3227754e3ca93f356fcfa884d0_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128281b7f3227754e3ca93f356fcfa884d0_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,350 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282c4e503b1c9a47f9890d38440bbaf4e1_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282c4e503b1c9a47f9890d38440bbaf4e1_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,351 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112832980fef216a400b826bc47e915b953e_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112832980fef216a400b826bc47e915b953e_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,351 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112859f9a09e16ee4b8c98e44d8cc07a1128_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112859f9a09e16ee4b8c98e44d8cc07a1128_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,352 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128610eff1d93b0483985a444251965eee2_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128610eff1d93b0483985a444251965eee2_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,353 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112897553c97673c4352aa947d9000ffc659_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112897553c97673c4352aa947d9000ffc659_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,354 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289756c4fafa0d4ec8a3bc3a909affdcbc_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289756c4fafa0d4ec8a3bc3a909affdcbc_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,355 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a3230ff09ec24eaeb435685130b0dbd6_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a3230ff09ec24eaeb435685130b0dbd6_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,356 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b84570ee38af4c7681831360b0e15a5c_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b84570ee38af4c7681831360b0e15a5c_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,357 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bc8733421901475194ec50c88e627ae1_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bc8733421901475194ec50c88e627ae1_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,358 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128cc9c32a2ae564b0e8ced78124d27af29_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128cc9c32a2ae564b0e8ced78124d27af29_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,359 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128cd9c1bc6a004421ebae61d58e4f95ff0_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128cd9c1bc6a004421ebae61d58e4f95ff0_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,360 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128db5d2f6aec4f44399b9823a374657ed5_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128db5d2f6aec4f44399b9823a374657ed5_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,361 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dd9270f6be3f4625988855483dc8d585_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dd9270f6be3f4625988855483dc8d585_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,362 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fb6f64e0be6242628187d0e4685a74e8_3bd4185211d8fb1e40f0aca1aa4a4576 to hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fb6f64e0be6242628187d0e4685a74e8_3bd4185211d8fb1e40f0aca1aa4a4576 2024-11-28T02:56:48,363 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T02:56:48,365 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:48,367 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T02:56:48,369 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T02:56:48,370 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:48,370 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T02:56:48,371 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732762608370"}]},"ts":"9223372036854775807"} 2024-11-28T02:56:48,373 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T02:56:48,373 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3bd4185211d8fb1e40f0aca1aa4a4576, NAME => 'TestAcidGuarantees,,1732762572423.3bd4185211d8fb1e40f0aca1aa4a4576.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T02:56:48,373 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T02:56:48,373 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732762608373"}]},"ts":"9223372036854775807"} 2024-11-28T02:56:48,375 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T02:56:48,375 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T02:56:48,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-28T02:56:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32823 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T02:56:48,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T02:56:48,437 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T02:56:48,447 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 251), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=359 (was 481), ProcessCount=11 (was 11), AvailableMemoryMB=8017 (was 7231) - AvailableMemoryMB LEAK? - 2024-11-28T02:56:48,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T02:56:48,447 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:56:48,447 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:56:48,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:48,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:48,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:56:48,447 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T02:56:48,447 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1484730631, stopped=false 2024-11-28T02:56:48,448 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=978bd30859c3,32823,1732762419008 2024-11-28T02:56:48,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T02:56:48,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T02:56:48,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:56:48,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:56:48,501 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T02:56:48,501 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:56:48,501 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:56:48,501 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T02:56:48,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:48,501 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '978bd30859c3,41631,1732762419946' ***** 2024-11-28T02:56:48,502 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T02:56:48,502 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T02:56:48,503 INFO [RS:0;978bd30859c3:41631 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T02:56:48,503 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T02:56:48,503 INFO [RS:0;978bd30859c3:41631 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T02:56:48,504 INFO [RS:0;978bd30859c3:41631 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T02:56:48,504 INFO [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(959): stopping server 978bd30859c3,41631,1732762419946 2024-11-28T02:56:48,504 INFO [RS:0;978bd30859c3:41631 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T02:56:48,504 INFO [RS:0;978bd30859c3:41631 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;978bd30859c3:41631. 2024-11-28T02:56:48,504 DEBUG [RS:0;978bd30859c3:41631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:56:48,504 DEBUG [RS:0;978bd30859c3:41631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:48,504 INFO [RS:0;978bd30859c3:41631 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T02:56:48,504 INFO [RS:0;978bd30859c3:41631 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T02:56:48,504 INFO [RS:0;978bd30859c3:41631 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T02:56:48,504 INFO [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T02:56:48,505 INFO [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T02:56:48,505 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-28T02:56:48,506 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T02:56:48,506 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T02:56:48,506 INFO [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T02:56:48,506 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T02:56:48,506 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T02:56:48,506 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T02:56:48,506 INFO [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-28T02:56:48,535 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/.tmp/info/5e02a1251ebd4e768b6ab53bc7e0e45a is 95, key is TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8./info:/1732762450101/DeleteFamily/seqid=0 2024-11-28T02:56:48,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742451_1627 (size=6514) 2024-11-28T02:56:48,567 INFO [regionserver/978bd30859c3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T02:56:48,706 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T02:56:48,906 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T02:56:48,940 INFO [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/.tmp/info/5e02a1251ebd4e768b6ab53bc7e0e45a 2024-11-28T02:56:48,958 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/.tmp/ns/7ffe146e59ee4c33a581ca8273ca6342 is 93, key is TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8./ns:/1732762450054/DeleteFamily/seqid=0 2024-11-28T02:56:48,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742452_1628 (size=5906) 2024-11-28T02:56:49,106 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T02:56:49,306 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T02:56:49,362 INFO [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/.tmp/ns/7ffe146e59ee4c33a581ca8273ca6342 2024-11-28T02:56:49,385 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/.tmp/rep_barrier/d7fa7ab19c4a4c0fba03be06ec3d1c87 is 102, key is TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8./rep_barrier:/1732762450054/DeleteFamily/seqid=0 2024-11-28T02:56:49,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742453_1629 (size=6025) 2024-11-28T02:56:49,395 INFO [regionserver/978bd30859c3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-28T02:56:49,395 INFO [regionserver/978bd30859c3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-28T02:56:49,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T02:56:49,507 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T02:56:49,707 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T02:56:49,789 INFO [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/.tmp/rep_barrier/d7fa7ab19c4a4c0fba03be06ec3d1c87 2024-11-28T02:56:49,807 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/.tmp/table/abc2f1587b2a47b79e7d3e5cb7a72ce7 is 96, key is TestAcidGuarantees,,1732762423914.1fec00bb9c12dd4e6236acd1fe2329f8./table:/1732762450054/DeleteFamily/seqid=0 2024-11-28T02:56:49,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742454_1630 (size=5936) 2024-11-28T02:56:49,907 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T02:56:50,107 DEBUG [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T02:56:50,210 INFO [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/.tmp/table/abc2f1587b2a47b79e7d3e5cb7a72ce7 2024-11-28T02:56:50,214 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/.tmp/info/5e02a1251ebd4e768b6ab53bc7e0e45a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/info/5e02a1251ebd4e768b6ab53bc7e0e45a 2024-11-28T02:56:50,217 INFO [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/info/5e02a1251ebd4e768b6ab53bc7e0e45a, entries=12, sequenceid=90, filesize=6.4 K 2024-11-28T02:56:50,218 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/.tmp/ns/7ffe146e59ee4c33a581ca8273ca6342 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/ns/7ffe146e59ee4c33a581ca8273ca6342 2024-11-28T02:56:50,221 INFO [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/ns/7ffe146e59ee4c33a581ca8273ca6342, entries=8, sequenceid=90, filesize=5.8 K 2024-11-28T02:56:50,222 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/.tmp/rep_barrier/d7fa7ab19c4a4c0fba03be06ec3d1c87 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/rep_barrier/d7fa7ab19c4a4c0fba03be06ec3d1c87 2024-11-28T02:56:50,225 INFO [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/rep_barrier/d7fa7ab19c4a4c0fba03be06ec3d1c87, entries=6, sequenceid=90, filesize=5.9 K 2024-11-28T02:56:50,226 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/.tmp/table/abc2f1587b2a47b79e7d3e5cb7a72ce7 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/table/abc2f1587b2a47b79e7d3e5cb7a72ce7 2024-11-28T02:56:50,229 INFO [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/table/abc2f1587b2a47b79e7d3e5cb7a72ce7, entries=7, sequenceid=90, filesize=5.8 K 2024-11-28T02:56:50,230 INFO [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1724ms, sequenceid=90, compaction requested=false 2024-11-28T02:56:50,240 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-28T02:56:50,240 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T02:56:50,241 INFO [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T02:56:50,241 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732762608505Running coprocessor pre-close hooks at 1732762608505Disabling compacts and flushes for region at 1732762608505Disabling writes for close at 1732762608506 (+1 ms)Obtaining lock to block concurrent updates at 1732762608506Preparing flush snapshotting stores in 1588230740 at 1732762608506Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732762608506Flushing stores of hbase:meta,,1.1588230740 at 1732762608507 (+1 ms)Flushing 1588230740/info: creating writer at 1732762608507Flushing 1588230740/info: appending metadata at 1732762608535 (+28 ms)Flushing 1588230740/info: closing flushed file at 1732762608535Flushing 1588230740/ns: creating writer at 1732762608944 (+409 ms)Flushing 1588230740/ns: appending metadata at 1732762608957 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732762608957Flushing 1588230740/rep_barrier: creating writer at 1732762609365 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732762609384 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732762609384Flushing 1588230740/table: creating writer at 1732762609792 (+408 ms)Flushing 1588230740/table: appending metadata at 1732762609806 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732762609806Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5703d233: reopening flushed file at 1732762610213 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@411a96e4: reopening flushed file at 1732762610218 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dd039e5: reopening flushed file at 1732762610221 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20a76895: reopening flushed file at 1732762610225 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1724ms, sequenceid=90, compaction requested=false at 1732762610230 (+5 ms)Writing region close event to WAL at 1732762610237 (+7 ms)Running coprocessor post-close hooks at 1732762610240 (+3 ms)Closed at 1732762610240 2024-11-28T02:56:50,241 DEBUG [RS_CLOSE_META-regionserver/978bd30859c3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T02:56:50,307 INFO [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(976): stopping server 978bd30859c3,41631,1732762419946; all regions closed. 2024-11-28T02:56:50,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741834_1010 (size=25344) 2024-11-28T02:56:50,313 DEBUG [RS:0;978bd30859c3:41631 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/oldWALs 2024-11-28T02:56:50,313 INFO [RS:0;978bd30859c3:41631 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 978bd30859c3%2C41631%2C1732762419946.meta:.meta(num 1732762423471) 2024-11-28T02:56:50,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741832_1008 (size=15186884) 2024-11-28T02:56:50,318 DEBUG [RS:0;978bd30859c3:41631 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/oldWALs 2024-11-28T02:56:50,318 INFO [RS:0;978bd30859c3:41631 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 978bd30859c3%2C41631%2C1732762419946:(num 1732762422523) 2024-11-28T02:56:50,318 DEBUG [RS:0;978bd30859c3:41631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:56:50,318 INFO [RS:0;978bd30859c3:41631 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T02:56:50,318 INFO [RS:0;978bd30859c3:41631 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T02:56:50,318 INFO [RS:0;978bd30859c3:41631 {}] hbase.ChoreService(370): Chore service for: regionserver/978bd30859c3:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T02:56:50,318 INFO [RS:0;978bd30859c3:41631 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T02:56:50,318 INFO [regionserver/978bd30859c3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T02:56:50,319 INFO [RS:0;978bd30859c3:41631 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41631 2024-11-28T02:56:50,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/978bd30859c3,41631,1732762419946 2024-11-28T02:56:50,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T02:56:50,349 INFO [RS:0;978bd30859c3:41631 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T02:56:50,350 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [978bd30859c3,41631,1732762419946] 2024-11-28T02:56:50,366 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/978bd30859c3,41631,1732762419946 already deleted, retry=false 2024-11-28T02:56:50,367 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 978bd30859c3,41631,1732762419946 expired; onlineServers=0 2024-11-28T02:56:50,367 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '978bd30859c3,32823,1732762419008' ***** 2024-11-28T02:56:50,367 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T02:56:50,367 INFO [M:0;978bd30859c3:32823 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T02:56:50,367 INFO [M:0;978bd30859c3:32823 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T02:56:50,367 DEBUG [M:0;978bd30859c3:32823 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T02:56:50,367 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T02:56:50,367 DEBUG [M:0;978bd30859c3:32823 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T02:56:50,367 DEBUG [master/978bd30859c3:0:becomeActiveMaster-HFileCleaner.large.0-1732762422130 {}] cleaner.HFileCleaner(306): Exit Thread[master/978bd30859c3:0:becomeActiveMaster-HFileCleaner.large.0-1732762422130,5,FailOnTimeoutGroup] 2024-11-28T02:56:50,367 DEBUG [master/978bd30859c3:0:becomeActiveMaster-HFileCleaner.small.0-1732762422131 {}] cleaner.HFileCleaner(306): Exit Thread[master/978bd30859c3:0:becomeActiveMaster-HFileCleaner.small.0-1732762422131,5,FailOnTimeoutGroup] 2024-11-28T02:56:50,367 INFO [M:0;978bd30859c3:32823 {}] hbase.ChoreService(370): Chore service for: master/978bd30859c3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T02:56:50,367 INFO [M:0;978bd30859c3:32823 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T02:56:50,368 DEBUG [M:0;978bd30859c3:32823 {}] master.HMaster(1795): Stopping service threads 2024-11-28T02:56:50,368 INFO [M:0;978bd30859c3:32823 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T02:56:50,368 INFO [M:0;978bd30859c3:32823 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T02:56:50,368 ERROR [M:0;978bd30859c3:32823 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1228365231) connection to localhost/127.0.0.1:35935 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:35935,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-28T02:56:50,369 INFO [M:0;978bd30859c3:32823 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T02:56:50,369 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T02:56:50,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T02:56:50,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T02:56:50,375 DEBUG [M:0;978bd30859c3:32823 {}] zookeeper.ZKUtil(347): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T02:56:50,375 WARN [M:0;978bd30859c3:32823 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T02:56:50,375 INFO [M:0;978bd30859c3:32823 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/.lastflushedseqids 2024-11-28T02:56:50,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742455_1631 (size=72) 2024-11-28T02:56:50,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T02:56:50,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41631-0x10180a5ce790001, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T02:56:50,460 INFO [RS:0;978bd30859c3:41631 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T02:56:50,460 INFO [RS:0;978bd30859c3:41631 {}] regionserver.HRegionServer(1031): Exiting; stopping=978bd30859c3,41631,1732762419946; zookeeper connection closed. 2024-11-28T02:56:50,460 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6934186e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6934186e 2024-11-28T02:56:50,461 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-28T02:56:50,785 INFO [M:0;978bd30859c3:32823 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T02:56:50,785 INFO [M:0;978bd30859c3:32823 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T02:56:50,785 DEBUG [M:0;978bd30859c3:32823 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T02:56:50,785 INFO [M:0;978bd30859c3:32823 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:56:50,785 DEBUG [M:0;978bd30859c3:32823 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:56:50,785 DEBUG [M:0;978bd30859c3:32823 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T02:56:50,785 DEBUG [M:0;978bd30859c3:32823 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:56:50,785 INFO [M:0;978bd30859c3:32823 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=774.09 KB heapSize=948.02 KB 2024-11-28T02:56:50,808 DEBUG [M:0;978bd30859c3:32823 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/83308bf2fb734b8095b2305fc9828007 is 82, key is hbase:meta,,1/info:regioninfo/1732762423558/Put/seqid=0 2024-11-28T02:56:50,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742456_1632 (size=5672) 2024-11-28T02:56:51,212 INFO [M:0;978bd30859c3:32823 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2144 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/83308bf2fb734b8095b2305fc9828007 2024-11-28T02:56:51,236 DEBUG [M:0;978bd30859c3:32823 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1ab87654595041898557746ce0d77d1a is 2480, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1732762453279/Put/seqid=0 2024-11-28T02:56:51,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742457_1633 (size=46119) 2024-11-28T02:56:51,640 INFO [M:0;978bd30859c3:32823 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=773.53 KB at sequenceid=2144 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1ab87654595041898557746ce0d77d1a 2024-11-28T02:56:51,644 INFO [M:0;978bd30859c3:32823 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1ab87654595041898557746ce0d77d1a 2024-11-28T02:56:51,666 DEBUG [M:0;978bd30859c3:32823 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5301306ad32a41ffa2f03abb98b01c8e is 69, key is 978bd30859c3,41631,1732762419946/rs:state/1732762422241/Put/seqid=0 2024-11-28T02:56:51,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073742458_1634 (size=5156) 2024-11-28T02:56:52,085 INFO [M:0;978bd30859c3:32823 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2144 (bloomFilter=true), to=hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5301306ad32a41ffa2f03abb98b01c8e 2024-11-28T02:56:52,093 DEBUG [M:0;978bd30859c3:32823 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/83308bf2fb734b8095b2305fc9828007 as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/83308bf2fb734b8095b2305fc9828007 2024-11-28T02:56:52,098 INFO [M:0;978bd30859c3:32823 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/83308bf2fb734b8095b2305fc9828007, entries=8, sequenceid=2144, filesize=5.5 K 2024-11-28T02:56:52,098 DEBUG [M:0;978bd30859c3:32823 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1ab87654595041898557746ce0d77d1a as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1ab87654595041898557746ce0d77d1a 2024-11-28T02:56:52,102 INFO [M:0;978bd30859c3:32823 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1ab87654595041898557746ce0d77d1a 2024-11-28T02:56:52,102 INFO [M:0;978bd30859c3:32823 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1ab87654595041898557746ce0d77d1a, entries=174, sequenceid=2144, filesize=45.0 K 2024-11-28T02:56:52,103 DEBUG [M:0;978bd30859c3:32823 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5301306ad32a41ffa2f03abb98b01c8e as hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5301306ad32a41ffa2f03abb98b01c8e 2024-11-28T02:56:52,107 INFO [M:0;978bd30859c3:32823 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35935/user/jenkins/test-data/7a6e761c-3736-dd72-e80b-355bf5824381/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5301306ad32a41ffa2f03abb98b01c8e, entries=1, sequenceid=2144, filesize=5.0 K 2024-11-28T02:56:52,107 INFO [M:0;978bd30859c3:32823 {}] regionserver.HRegion(3140): Finished flush of dataSize ~774.09 KB/792667, heapSize ~947.72 KB/970464, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1322ms, sequenceid=2144, compaction requested=false 2024-11-28T02:56:52,109 INFO [M:0;978bd30859c3:32823 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:56:52,109 DEBUG [M:0;978bd30859c3:32823 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732762610785Disabling compacts and flushes for region at 1732762610785Disabling writes for close at 1732762610785Obtaining lock to block concurrent updates at 1732762610785Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732762610785Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=792667, getHeapSize=970704, getOffHeapSize=0, getCellsCount=2343 at 1732762610786 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732762610786Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732762610786Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732762610807 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732762610807Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732762611217 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732762611236 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732762611236Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732762611644 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732762611665 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732762611665Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57709d3e: reopening flushed file at 1732762612091 (+426 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70fda817: reopening flushed file at 1732762612098 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3236e791: reopening flushed file at 1732762612103 (+5 ms)Finished flush of dataSize ~774.09 KB/792667, heapSize ~947.72 KB/970464, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1322ms, sequenceid=2144, compaction requested=false at 1732762612107 (+4 ms)Writing region close event to WAL at 1732762612109 (+2 ms)Closed at 1732762612109 2024-11-28T02:56:52,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44799 is added to blk_1073741830_1006 (size=933242) 2024-11-28T02:56:52,111 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T02:56:52,111 INFO [M:0;978bd30859c3:32823 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T02:56:52,112 INFO [M:0;978bd30859c3:32823 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32823 2024-11-28T02:56:52,112 INFO [M:0;978bd30859c3:32823 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T02:56:52,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T02:56:52,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32823-0x10180a5ce790000, quorum=127.0.0.1:49459, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T02:56:52,233 INFO [M:0;978bd30859c3:32823 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T02:56:52,240 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T02:56:52,242 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T02:56:52,243 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T02:56:52,243 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T02:56:52,243 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/hadoop.log.dir/,STOPPED} 2024-11-28T02:56:52,247 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-28T02:56:52,247 WARN [BP-872746337-172.17.0.2-1732762414943 heartbeating to localhost/127.0.0.1:35935 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-28T02:56:52,247 WARN [BP-872746337-172.17.0.2-1732762414943 heartbeating to localhost/127.0.0.1:35935 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-872746337-172.17.0.2-1732762414943 (Datanode Uuid a695dfb8-90fe-4dfb-9b42-b7f21223e480) service to localhost/127.0.0.1:35935 2024-11-28T02:56:52,247 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-28T02:56:52,249 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/cluster_ae8f2237-5101-485d-8e1c-1b2973f77a47/data/data1/current/BP-872746337-172.17.0.2-1732762414943 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T02:56:52,250 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/cluster_ae8f2237-5101-485d-8e1c-1b2973f77a47/data/data2/current/BP-872746337-172.17.0.2-1732762414943 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T02:56:52,250 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-28T02:56:52,257 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T02:56:52,258 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T02:56:52,258 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T02:56:52,258 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T02:56:52,258 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2f689053-9728-b8ce-3c94-a748925d484e/hadoop.log.dir/,STOPPED} 2024-11-28T02:56:52,272 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T02:56:52,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down